[llvm] [AMDGPU] Use S_CMP to lower a copy of a lane mask to SCC (PR #221445)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 06:28:14 PDT 2026
https://github.com/jayfoad updated https://github.com/llvm/llvm-project/pull/221445
>From 597fb564e415cb8b0783d7a83c516cbbf997ccfb Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Fri, 4 Sep 2026 10:13:25 +0100
Subject: [PATCH 1/5] [AMDGPU] Use S_CMP to lower a copy of a lane mask to SCC
SIFixSGPRCopies lowers a copy of a lane mask to SCC as an AND with EXEC,
whose destination register is created by the pass and is always dead.
The AND with EXEC is only needed because SCC has to be "any active lane
is set". When the lane mask already has 0 in the bits of all inactive
lanes, that is just "the mask is non-zero", which S_CMP computes without
needing a destination register. In some cases the S_CMP can be optimized
away later by SIInstrInfo::optimizeCompareInstr.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 2 +
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 34 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 100 +
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 8 +
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 1775 +++++++++--------
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 746 +++----
llvm/test/CodeGen/AMDGPU/anyext.ll | 4 +-
.../atomic_optimizations_local_pointer.ll | 192 +-
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 4 +-
.../CodeGen/AMDGPU/copy-to-reg-scc-clobber.ll | 16 +-
.../CodeGen/AMDGPU/extract-subvector-16bit.ll | 12 +-
llvm/test/CodeGen/AMDGPU/fcmp.f16.ll | 258 +--
llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll | 176 +-
.../AMDGPU/fix-sgpr-copies-scc-cmp.mir | 59 +
llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll | 12 +-
.../CodeGen/AMDGPU/fneg-modifier-casting.ll | 51 +-
llvm/test/CodeGen/AMDGPU/fp-classify.ll | 56 +-
llvm/test/CodeGen/AMDGPU/fp_to_sint.ll | 8 +-
llvm/test/CodeGen/AMDGPU/fp_to_uint.ll | 8 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll | 46 +-
llvm/test/CodeGen/AMDGPU/fptosi.f16.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fptoui.f16.ll | 6 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 30 +-
llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll | 4 +-
llvm/test/CodeGen/AMDGPU/literal64.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll | 58 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll | 10 +-
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll | 16 +-
llvm/test/CodeGen/AMDGPU/llvm.log.ll | 354 ++--
llvm/test/CodeGen/AMDGPU/llvm.log10.ll | 354 ++--
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 132 +-
llvm/test/CodeGen/AMDGPU/max.ll | 8 +-
llvm/test/CodeGen/AMDGPU/min.ll | 40 +-
llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll | 6 +-
llvm/test/CodeGen/AMDGPU/or.ll | 2 +-
.../AMDGPU/pseudo-scalar-transcendental.ll | 13 +-
llvm/test/CodeGen/AMDGPU/rsq.f64.ll | 6 +-
llvm/test/CodeGen/AMDGPU/saddo.ll | 22 +-
llvm/test/CodeGen/AMDGPU/scmp.ll | 20 +-
.../test/CodeGen/AMDGPU/setcc-f64-hi32mask.ll | 22 +-
llvm/test/CodeGen/AMDGPU/ssubo.ll | 14 +-
llvm/test/CodeGen/AMDGPU/ucmp.ll | 20 +-
.../umin-sub-to-usubo-select-combine.ll | 4 +-
llvm/test/CodeGen/AMDGPU/wave32.ll | 4 +-
45 files changed, 2468 insertions(+), 2270 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index af14066c1bd468..9bd8acd945c7e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -32,6 +32,7 @@ class LaneMaskConstants {
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
const unsigned CMovOpc;
+ const unsigned CmpLgOpc; // GFX8+ only in wave64
const unsigned CSelectOpc;
const unsigned MovOpc;
const unsigned MovTermOpc;
@@ -61,6 +62,7 @@ class LaneMaskConstants {
: AMDGPU::S_AND_SAVEEXEC_B64_term),
BfmOpc(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64),
CMovOpc(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
+ CmpLgOpc(IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64),
CSelectOpc(IsWave32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64),
MovOpc(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
MovTermOpc(IsWave32 ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term),
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index b0c896b1a122ad..a3869fec8910fc 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1184,8 +1184,11 @@ void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
}
void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
- const AMDGPU::LaneMaskConstants &LMC =
- AMDGPU::LaneMaskConstants::get(MF.getSubtarget<GCNSubtarget>());
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+ // S_CMP_LG_U64 is only available on GFX8+. S_CMP_LG_U32 always is, and
+ // wave32 implies GFX10+ anyway.
+ bool HasCmp = ST.isWave32() || ST.hasScalarCompareEq64();
for (MachineBasicBlock &MBB : MF) {
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
++I) {
@@ -1209,12 +1212,27 @@ void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
continue;
}
if (DstReg == AMDGPU::SCC) {
- Register Tmp = MRI->createVirtualRegister(TRI->getBoolRC());
- I = BuildMI(*MI.getParent(), std::next(MachineBasicBlock::iterator(MI)),
- MI.getDebugLoc(), TII->get(LMC.AndOpc))
- .addReg(Tmp, getDefRegState(true))
- .addReg(SrcReg)
- .addReg(LMC.ExecReg);
+ // Both lowerings below read the whole of SrcReg.
+ assert(!MI.getOperand(1).getSubReg() &&
+ "cannot lower a copy of a subregister to SCC");
+ MachineBasicBlock::iterator InsPt =
+ std::next(MachineBasicBlock::iterator(MI));
+ if (HasCmp && TII->isMaskedByExec(SrcReg, MI, *MRI)) {
+ // The source already has 0 in the bits of all inactive lanes, so
+ // SCC is just "source is non-zero". S_CMP computes that without
+ // needing a destination register.
+ I = BuildMI(*MI.getParent(), InsPt, MI.getDebugLoc(),
+ TII->get(LMC.CmpLgOpc))
+ .addReg(SrcReg)
+ .addImm(0);
+ } else {
+ Register Tmp = MRI->createVirtualRegister(TRI->getBoolRC());
+ I = BuildMI(*MI.getParent(), InsPt, MI.getDebugLoc(),
+ TII->get(LMC.AndOpc))
+ .addReg(Tmp, getDefRegState(true))
+ .addReg(SrcReg)
+ .addReg(LMC.ExecReg);
+ }
MI.eraseFromParent();
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 11b16b73933c1f..37c407f753eeff 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5037,6 +5037,106 @@ bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
return pseudoToMCOpcode(Op32) != -1;
}
+/// Return true if \p MI is a VALU comparison, i.e. an instruction that writes
+/// a lane mask with one bit per lane, and zeroes the bits of lanes that were
+/// inactive when it executed. The VOP3 (_e64) encodings are not themselves
+/// marked as VOPC, so also check the VOPC (_e32) equivalent.
+///
+/// TODO: Also handle the sdst result of V_ADD_CO_U32 and V_SUB_CO_U32 amd
+/// V_DIV_SCALE_F32.
+static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI) {
+ if (TII.isVOPC(MI))
+ return true;
+ int Op32 = AMDGPU::getVOPe32(MI.getOpcode());
+ return Op32 != -1 && TII.isVOPC(Op32);
+}
+
+/// Worker for SIInstrInfo::isMaskedByExec. Every definition the result depends
+/// on must be in \p MBB, and is added to \p Defs so that the caller can check
+/// that EXEC is not modified between it and the use. Definitions are removed
+/// again when a branch of the search fails, so that on success \p Defs only
+/// holds definitions the result really depends on.
+static bool isMaskedByExecImpl(const SIInstrInfo &TII,
+ const AMDGPU::LaneMaskConstants &LMC,
+ Register Reg, const MachineBasicBlock *MBB,
+ SmallSetVector<const MachineInstr *, 8> &Defs,
+ const MachineRegisterInfo &MRI, unsigned Depth) {
+ // EXEC itself is trivially masked by EXEC.
+ if (Reg == LMC.ExecReg)
+ return true;
+
+ if (Depth == 0 || !Reg.isVirtual())
+ return false;
+
+ // Only look at definitions that can execute under the same EXEC mask as the
+ // use. Note that this also rules out instructions that write EXEC
+ // themselves, such as V_CMPX, since those would be found by the caller's
+ // scan for writes to EXEC.
+ const MachineInstr *Def = MRI.getVRegDef(Reg);
+ if (Def->getParent() != MBB)
+ return false;
+
+ size_t NumDefs = Defs.size();
+ Defs.insert(Def);
+
+ if (isVCmp(TII, *Def))
+ return true;
+
+ // Recurse into an operand, which must be a whole register to say anything
+ // about the whole lane mask. A failed call rolls back its own additions.
+ auto Recurse = [&](unsigned OpIdx) {
+ const MachineOperand &MO = Def->getOperand(OpIdx);
+ return MO.isReg() && !MO.getSubReg() &&
+ isMaskedByExecImpl(TII, LMC, MO.getReg(), MBB, Defs, MRI, Depth - 1);
+ };
+
+ unsigned Opc = Def->getOpcode();
+ if (Opc == AMDGPU::COPY && Recurse(1))
+ return true;
+ // AND only needs one masked operand, because a zero bit in either operand
+ // forces a zero bit in the result.
+ if (Opc == LMC.AndOpc && (Recurse(1) || Recurse(2)))
+ return true;
+ // Likewise ANDN2, but only for its first operand.
+ if (Opc == LMC.AndN2Opc && Recurse(1))
+ return true;
+ if ((Opc == LMC.OrOpc || Opc == LMC.XorOpc) && Recurse(1) && Recurse(2))
+ return true;
+
+ // Roll back, so that a caller that succeeds by another route is not left
+ // depending on definitions it does not actually use.
+ while (Defs.size() > NumDefs)
+ Defs.pop_back();
+ return false;
+}
+
+bool SIInstrInfo::isMaskedByExec(Register Reg, const MachineInstr &Use,
+ const MachineRegisterInfo &MRI) const {
+ assert(MRI.isSSA() && "isMaskedByExec requires SSA form");
+ const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+ const MachineBasicBlock *MBB = Use.getParent();
+
+ // Maximum depth of the def-use walk.
+ constexpr unsigned MaxDepth = 6;
+ SmallSetVector<const MachineInstr *, 8> Defs;
+ if (!isMaskedByExecImpl(*this, LMC, Reg, MBB, Defs, MRI, MaxDepth))
+ return false;
+
+ // Walk back from Use, which every definition dominates, and check that all
+ // of them are reached before any write to EXEC. Otherwise the mask may have
+ // been computed under an EXEC that is not a subset of the one in effect at
+ // Use, and so may have bits set for lanes that are now inactive.
+ unsigned NumDefsToFind = Defs.size();
+ for (const MachineInstr &Prev : reverse(
+ make_range(MBB->begin(), MachineBasicBlock::const_iterator(Use)))) {
+ if (Prev.modifiesRegister(LMC.ExecReg, &RI))
+ return false;
+ if (Defs.contains(&Prev) && --NumDefsToFind == 0)
+ break;
+ }
+ return NumDefsToFind == 0;
+}
+
bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
// The src0_modifier operand is present on all instructions
// that have modifiers.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index be31e17b8a1604..0d2f4c7f02ae34 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1382,6 +1382,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// This function will return false if you pass it a 32-bit instruction.
bool hasVALU32BitEncoding(unsigned Opcode) const;
+ /// Return true if \p Reg is a lane mask that already has 0 in every bit
+ /// corresponding to a lane that is inactive in EXEC where \p Use executes,
+ /// so that ANDing it with EXEC there would be a no-op. Only definitions in
+ /// the same basic block as \p Use, and not separated from it by a write to
+ /// EXEC, are considered. Requires SSA form.
+ bool isMaskedByExec(Register Reg, const MachineInstr &Use,
+ const MachineRegisterInfo &MRI) const;
+
bool physRegUsesConstantBus(const MachineOperand &Reg) const;
bool regUsesConstantBus(const MachineOperand &Reg,
const MachineRegisterInfo &MRI) const;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 7be6ad4575d694..54627424ab4aa8 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -18935,7 +18935,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
@@ -19822,7 +19822,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
@@ -20603,11 +20603,11 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:4
@@ -20644,9 +20644,9 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB15_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -20710,19 +20710,19 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s4, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v176, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v177, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -20895,7 +20895,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -20935,8 +20936,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -21182,9 +21183,9 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
+; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:28
+; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v181, off, s32 offset:12
@@ -21223,9 +21224,9 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB15_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -21289,7 +21290,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -21301,7 +21302,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v176, v166, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -21474,7 +21475,8 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v176
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -21515,7 +21517,7 @@ define inreg <32 x i32> @bitcast_v128i8_to_v32i32_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -54945,7 +54947,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
@@ -55832,7 +55834,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
@@ -56613,11 +56615,11 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:4
@@ -56654,9 +56656,9 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -56720,19 +56722,19 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s4, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v176, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v177, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -56905,7 +56907,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -56945,8 +56948,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -57192,9 +57195,9 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
+; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:28
+; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v181, off, s32 offset:12
@@ -57233,9 +57236,9 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -57299,7 +57302,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -57311,7 +57314,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v176, v166, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -57484,7 +57487,8 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v176
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -57525,7 +57529,7 @@ define inreg <32 x float> @bitcast_v128i8_to_v32f32_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -88960,7 +88964,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
@@ -89847,7 +89851,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
@@ -90628,11 +90632,11 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:4
@@ -90669,9 +90673,9 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -90735,19 +90739,19 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s4, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v176, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v177, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -90920,7 +90924,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -90960,8 +90965,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -91207,9 +91212,9 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
+; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:28
+; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v181, off, s32 offset:12
@@ -91248,9 +91253,9 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -91314,7 +91319,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -91326,7 +91331,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v176, v166, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -91499,7 +91504,8 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v176
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -91540,7 +91546,7 @@ define inreg <16 x i64> @bitcast_v128i8_to_v16i64_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -121986,7 +121992,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; VI-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; VI-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; VI-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; VI-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
@@ -122873,7 +122879,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX9-NEXT: buffer_load_ushort v52, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v53, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v54, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(47)
; GFX9-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
; GFX9-NEXT: s_waitcnt vmcnt(47)
@@ -123654,11 +123660,11 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:8
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:4
@@ -123695,9 +123701,9 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB75_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -123761,19 +123767,19 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, s4, v179, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v15
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v177, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v181, v176, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v177, v166, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -123946,7 +123952,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v177
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -123986,8 +123993,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v176, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -124233,9 +124240,9 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: scratch_load_u16 v162, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v165, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v166, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:24
+; GFX11-FAKE16-NEXT: scratch_load_u16 v176, off, s32 offset:28
+; GFX11-FAKE16-NEXT: scratch_load_u16 v167, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v180, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v177, off, s32 offset:16
; GFX11-FAKE16-NEXT: scratch_load_u16 v181, off, s32 offset:12
@@ -124274,9 +124281,9 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB75_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v10, 0xc0c0004
@@ -124340,7 +124347,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s8, s7, v10
; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, v11, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v12, v180, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, v13, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -124352,7 +124359,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, v13, v14
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v163, v151, 0xc0c0004
; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, v15, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v167, v166, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v176, v166, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v164, v160, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
@@ -124525,7 +124532,8 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_perm_b32 v8, s44, s43, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v9, s42, s41, v10
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v13
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v167
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v15, 3, v176
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 3, v165
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v9
@@ -124566,7 +124574,7 @@ define inreg <16 x double> @bitcast_v128i8_to_v16f64_scalar(<128 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v17
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v178, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v13, v13, v177, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v176, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v14, v167, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v18
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v12
@@ -147354,7 +147362,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
@@ -148243,7 +148251,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(41)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32
@@ -149028,129 +149036,129 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:292
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:260
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:240
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:224
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:208
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:204
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:172
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:84
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v27
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v25
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v24
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v19
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v13
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB89_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0x6050400
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v70, v114, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v82, v114, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v32, v38, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v33, v50, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v11
@@ -149161,92 +149169,92 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s9, s4, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s9, s5, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s40, s8, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s14, s7, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s13, s8, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s4, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s12, s6, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s12, s7, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s5, s56, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s6, s56, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s61, s45, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s43, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s72, s45, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s42, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s60, s15, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, s13, s46, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s60, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, s14, s46, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s72, s58, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s57, s63, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s57, s62, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s59, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s59, s43, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s10, s41, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, s47, s62, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s10, s40, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, s47, s61, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v37, v68, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s42, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v37, v69, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s41, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v86, s74, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v53, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v134, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v135, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v66, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v39, v83, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v71, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v39, v166, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v70, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v99, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v69, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v98, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v68, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v97, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v84, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v87, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v83, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v103, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v98, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v102, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v96, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v116, v67, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v113, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v115, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v112, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v130, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v118, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v129, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v84, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v64, v117, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v64, v116, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v82, v131, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v133, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v81, v130, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v147, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v87, v144, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v146, v99, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v134, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v151, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v101, v149, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v150, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v100, v148, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v24.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v119, v164, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v132, v162, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v118, v163, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v131, v161, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v25.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v176, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v102, v160, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v176, v147, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v101, v151, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v128, v167, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v129, v165, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v119, v167, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v128, v164, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v27.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v177, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v183, v133, v166, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v177, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v183, v132, v165, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v150, v179, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v149, v179, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v29.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v181, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v181, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.l
; GFX11-TRUE16-NEXT: v_perm_b32 v30, v178, v182, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v31.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v180, v161, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v180, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v183.l
; GFX11-TRUE16-NEXT: s_branch .LBB89_3
; GFX11-TRUE16-NEXT: .LBB89_2:
@@ -149261,98 +149269,98 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB89_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v180
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v182
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v165
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v164
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v181
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v161, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v132, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v178, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v31, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v133, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v129, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v160
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v151
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v30, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v179
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v162
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v129, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v161
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v128, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v177
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v102, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v101, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v148
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v131, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v145, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v167
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v176
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v128, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v101, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v147, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v130
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v163
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v117
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v116
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v118, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v150
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v117
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v147
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v146
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v112
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v135
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v84, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v99, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v133
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v96
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v144
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v83
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v129
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v71
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v68
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v70
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v103
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v102
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v97
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v87
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v99
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v98
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v0
@@ -149360,81 +149368,82 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v83
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v114
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v68
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v86, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s42, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s62, s47, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s41, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s61, s47, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s59, s44, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s59, s43, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s72, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s43, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s56, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s42, s11, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s56, s6, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s12, s6, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s12, s7, v1
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s41, s10, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s46, s13, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s46, s14, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s60, s15, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s61, s45, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s60, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s45, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: s_add_i32 s9, s9, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s63, s57, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v48, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s9, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s9, s5, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s14, s7, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v54, s40, s8, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v54, s13, s8, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v67, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -149452,13 +149461,13 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v67
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v69
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v70
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v80
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v69.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v82.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v68.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v54.l
@@ -149479,15 +149488,15 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v52.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v84.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v87.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v101.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v132.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v102.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v129.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v133.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v100.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v131.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v101.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v128.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v132.l
; GFX11-TRUE16-NEXT: .LBB89_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -149553,34 +149562,34 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:116
; GFX11-FAKE16-NEXT: scratch_load_u16 v83, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:108
-; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:104
; GFX11-FAKE16-NEXT: scratch_load_u16 v71, off, s32 offset:100
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:96
; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:92
-; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:88
+; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:88
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:84
-; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:80
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:76
-; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:72
-; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:68
-; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:68
+; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:64
; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:52
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:44
+; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:44
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
+; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:12
+; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:4
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v30
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s61, v29
@@ -149595,7 +149604,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s46, v20
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s57, v19
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s60, v18
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v17
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v17
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s40, v16
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s47, v15
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s58, v14
@@ -149604,19 +149613,19 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s41, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s44, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v9
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v8
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v8
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s15, v7
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s42, v6
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v5
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v4
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB89_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -149624,7 +149633,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v32, v48, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v36, v49, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v19, v68, v55, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v19, v67, v54, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v2, s16, s17, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s20, s21, v9
@@ -149640,7 +149649,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s11, s8, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s13, s8, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s28, s29, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v7, 16, v6
; GFX11-FAKE16-NEXT: v_perm_b32 v7, s42, s15, v9
@@ -149648,7 +149657,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_perm_b32 v6, s6, s4, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_perm_b32 v10, s12, s9, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v10, s11, s9, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
; GFX11-FAKE16-NEXT: v_perm_b32 v12, s44, s41, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v8, 16, v5
@@ -149659,7 +149668,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_perm_b32 v7, s14, s10, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v12
; GFX11-FAKE16-NEXT: v_perm_b32 v12, s60, s57, v9
-; GFX11-FAKE16-NEXT: v_perm_b32 v13, s13, s40, v11
+; GFX11-FAKE16-NEXT: v_perm_b32 v13, s12, s40, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-FAKE16-NEXT: v_perm_b32 v16, s45, s56, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v14
@@ -149670,8 +149679,8 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s73, s72, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v16, 16, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v67, v51, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v53, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v66, v52, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v68, 0x6050400
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v15, 16, v12
; GFX11-FAKE16-NEXT: v_perm_b32 v11, s61, s63, v11
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -149679,18 +149688,18 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v14
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v33, v38, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v33, v39, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff, v12
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v17, 16, v14
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v18, 16, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v80, v66, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v80, v65, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v15, 16, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v65, v39, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v82, v64, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v64, v38, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v82, v55, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v34, v50, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v69, v52, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v20, v69, v51, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v15, 0xffff, v15
; GFX11-FAKE16-NEXT: v_and_b32_e32 v18, 0xffff, v18
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v96, v85, 0xc0c0004
@@ -149698,7 +149707,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v98, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v17, 16, v15
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v20, 16, v18
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v71, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v71, v53, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v20, 0xffff, v21
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v86, v81, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
@@ -149851,36 +149860,36 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v82
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v81, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v64, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v53, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v67
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v80
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v65
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v64
; GFX11-FAKE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v52, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v66, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v55, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v39, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v51, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v65, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v38, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v67
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v66
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v68
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v49
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v51, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v52, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v36, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v38
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -149895,19 +149904,19 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s62, s59, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s11, s11, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s73, s72, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s40, s13, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s40, s12, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s63, s61, v2
; GFX11-FAKE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-FAKE16-NEXT: s_add_i32 s60, s60, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s12, s9, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s11, s9, v2
; GFX11-FAKE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-FAKE16-NEXT: s_add_i32 s58, s58, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v3
@@ -149918,7 +149927,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s46, s43, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v5, s58, s47, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s11, s8, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s13, s8, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v50, s24, s25, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v65, s0, s1, v2
; GFX11-FAKE16-NEXT: s_add_i32 s44, s44, 3
@@ -149933,7 +149942,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s44, s41, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v3, s14, s10, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s42, s15, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v67, s2, s3, v2
@@ -149960,7 +149969,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v5, s6, s4, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s20, s21, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v66, s18, s19, v2
@@ -149976,7 +149985,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v35, 16, v33
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v32, 16, v36
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v54
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v53
; GFX11-FAKE16-NEXT: v_and_b32_e32 v33, 0xffff, v71
; GFX11-FAKE16-NEXT: v_and_b32_e32 v34, 0xffff, v18
; GFX11-FAKE16-NEXT: v_and_b32_e32 v35, 0xffff, v17
@@ -149987,9 +149996,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v64
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v51, 0xffff, v51
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v55, 16, v32
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v52, 16, v33
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v52
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v54, 16, v32
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v51, 16, v33
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v20, v20, 16, v34
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v21, v21, 16, v35
; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v99
@@ -150000,9 +150009,9 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v66, 16, v64
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v68
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v53, 16, v51
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v55, 16, v52
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v48
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v51, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
; GFX11-FAKE16-NEXT: v_and_b32_e32 v24, 0xffff, v24
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v22, v100, 16, v32
@@ -150018,7 +150027,7 @@ define inreg <64 x bfloat> @bitcast_v128i8_to_v64bf16_scalar(<128 x i8> inreg %a
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v6, 16, v65
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v49, 16, v48
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v10, 16, v50
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v51
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v52
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v15, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v16, 16, v38
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v19, v70, 16, v19
@@ -173648,7 +173657,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
@@ -174537,7 +174546,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(41)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32
@@ -175322,129 +175331,129 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:292
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:260
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:240
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:224
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:208
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:204
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:172
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:84
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v27
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v25
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v24
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v19
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v13
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB93_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0x6050400
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v70, v114, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v82, v114, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v32, v38, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v33, v50, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v11
@@ -175455,92 +175464,92 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s9, s4, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s9, s5, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s40, s8, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s14, s7, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s13, s8, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s4, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s12, s6, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s12, s7, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s5, s56, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s6, s56, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s61, s45, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s43, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s72, s45, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s42, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s60, s15, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, s13, s46, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s60, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, s14, s46, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s72, s58, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s57, s63, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s57, s62, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s59, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s59, s43, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s10, s41, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, s47, s62, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s10, s40, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, s47, s61, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v37, v68, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s42, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v37, v69, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s41, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v86, s74, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v53, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v134, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v135, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v66, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v39, v83, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v71, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v39, v166, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v70, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v99, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v69, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v98, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v68, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v97, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v84, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v87, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v83, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v103, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v98, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v102, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v96, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v116, v67, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v113, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v115, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v112, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v130, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v118, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v129, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v84, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v64, v117, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v64, v116, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v82, v131, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v133, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v81, v130, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v147, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v87, v144, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v146, v99, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v134, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v151, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v101, v149, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v150, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v100, v148, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v24.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v119, v164, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v132, v162, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v118, v163, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v131, v161, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v25.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v176, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v102, v160, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v176, v147, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v101, v151, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v128, v167, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v129, v165, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v119, v167, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v128, v164, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v27.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v177, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v183, v133, v166, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v177, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v183, v132, v165, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v150, v179, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v149, v179, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v29.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v181, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v181, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.l
; GFX11-TRUE16-NEXT: v_perm_b32 v30, v178, v182, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v31.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v180, v161, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v180, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v183.l
; GFX11-TRUE16-NEXT: s_branch .LBB93_3
; GFX11-TRUE16-NEXT: .LBB93_2:
@@ -175555,98 +175564,98 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB93_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v180
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v182
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v165
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v164
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v181
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v161, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v132, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v178, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v31, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v133, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v129, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v160
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v151
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v30, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v179
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v162
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v129, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v161
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v128, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v177
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v102, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v101, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v148
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v131, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v145, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v167
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v176
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v128, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v101, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v147, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v130
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v163
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v117
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v116
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v118, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v150
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v117
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v147
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v146
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v112
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v135
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v84, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v99, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v133
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v96
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v144
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v83
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v129
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v71
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v68
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v70
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v103
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v102
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v97
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v87
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v99
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v98
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v0
@@ -175654,81 +175663,82 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v83
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v114
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v68
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v86, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s42, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s62, s47, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s41, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s61, s47, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s59, s44, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s59, s43, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s72, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s43, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s56, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s42, s11, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s56, s6, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s12, s6, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s12, s7, v1
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s41, s10, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s46, s13, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s46, s14, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s60, s15, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s61, s45, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s60, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s45, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: s_add_i32 s9, s9, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s63, s57, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v48, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s9, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s9, s5, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s14, s7, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v54, s40, s8, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v54, s13, s8, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v67, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -175746,13 +175756,13 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v67
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v69
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v70
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v80
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v69.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v82.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v68.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v54.l
@@ -175773,15 +175783,15 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v52.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v84.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v87.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v101.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v132.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v102.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v129.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v133.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v100.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v131.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v101.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v128.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v132.l
; GFX11-TRUE16-NEXT: .LBB93_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -175847,34 +175857,34 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:116
; GFX11-FAKE16-NEXT: scratch_load_u16 v83, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:108
-; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:104
; GFX11-FAKE16-NEXT: scratch_load_u16 v71, off, s32 offset:100
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:96
; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:92
-; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:88
+; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:88
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:84
-; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:80
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:76
-; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:72
-; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:68
-; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:68
+; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:64
; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:52
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:44
+; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:44
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
+; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:12
+; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:4
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v30
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s61, v29
@@ -175889,7 +175899,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s46, v20
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s57, v19
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s60, v18
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v17
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v17
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s40, v16
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s47, v15
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s58, v14
@@ -175898,19 +175908,19 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s41, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s44, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v9
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v8
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v8
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s15, v7
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s42, v6
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v5
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v4
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB93_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -175918,7 +175928,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v32, v48, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v36, v49, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v19, v68, v55, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v19, v67, v54, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v2, s16, s17, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s20, s21, v9
@@ -175934,7 +175944,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s11, s8, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s13, s8, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s28, s29, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v7, 16, v6
; GFX11-FAKE16-NEXT: v_perm_b32 v7, s42, s15, v9
@@ -175942,7 +175952,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_perm_b32 v6, s6, s4, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_perm_b32 v10, s12, s9, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v10, s11, s9, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
; GFX11-FAKE16-NEXT: v_perm_b32 v12, s44, s41, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v8, 16, v5
@@ -175953,7 +175963,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_perm_b32 v7, s14, s10, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v12
; GFX11-FAKE16-NEXT: v_perm_b32 v12, s60, s57, v9
-; GFX11-FAKE16-NEXT: v_perm_b32 v13, s13, s40, v11
+; GFX11-FAKE16-NEXT: v_perm_b32 v13, s12, s40, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-FAKE16-NEXT: v_perm_b32 v16, s45, s56, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v14
@@ -175964,8 +175974,8 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s73, s72, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v16, 16, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v67, v51, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v53, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v66, v52, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v68, 0x6050400
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v15, 16, v12
; GFX11-FAKE16-NEXT: v_perm_b32 v11, s61, s63, v11
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -175973,18 +175983,18 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v14
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v33, v38, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v33, v39, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff, v12
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v17, 16, v14
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v18, 16, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v80, v66, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v80, v65, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v15, 16, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v65, v39, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v82, v64, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v64, v38, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v82, v55, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v34, v50, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v69, v52, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v20, v69, v51, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v15, 0xffff, v15
; GFX11-FAKE16-NEXT: v_and_b32_e32 v18, 0xffff, v18
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v96, v85, 0xc0c0004
@@ -175992,7 +176002,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v98, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v17, 16, v15
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v20, 16, v18
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v71, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v71, v53, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v20, 0xffff, v21
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v86, v81, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
@@ -176145,36 +176155,36 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v82
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v81, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v64, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v53, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v67
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v80
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v65
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v64
; GFX11-FAKE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v52, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v66, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v55, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v39, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v51, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v65, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v38, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v67
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v66
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v68
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v49
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v51, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v52, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v36, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v38
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -176189,19 +176199,19 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s62, s59, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s11, s11, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s73, s72, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s40, s13, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s40, s12, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s63, s61, v2
; GFX11-FAKE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-FAKE16-NEXT: s_add_i32 s60, s60, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s12, s9, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s11, s9, v2
; GFX11-FAKE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-FAKE16-NEXT: s_add_i32 s58, s58, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v3
@@ -176212,7 +176222,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s46, s43, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v5, s58, s47, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s11, s8, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s13, s8, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v50, s24, s25, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v65, s0, s1, v2
; GFX11-FAKE16-NEXT: s_add_i32 s44, s44, 3
@@ -176227,7 +176237,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s44, s41, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v3, s14, s10, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s42, s15, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v67, s2, s3, v2
@@ -176254,7 +176264,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v5, s6, s4, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s20, s21, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v66, s18, s19, v2
@@ -176270,7 +176280,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v35, 16, v33
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v32, 16, v36
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v54
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v53
; GFX11-FAKE16-NEXT: v_and_b32_e32 v33, 0xffff, v71
; GFX11-FAKE16-NEXT: v_and_b32_e32 v34, 0xffff, v18
; GFX11-FAKE16-NEXT: v_and_b32_e32 v35, 0xffff, v17
@@ -176281,9 +176291,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v64
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v51, 0xffff, v51
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v55, 16, v32
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v52, 16, v33
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v52
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v54, 16, v32
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v51, 16, v33
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v20, v20, 16, v34
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v21, v21, 16, v35
; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v99
@@ -176294,9 +176304,9 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v66, 16, v64
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v68
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v53, 16, v51
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v55, 16, v52
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v48
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v51, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
; GFX11-FAKE16-NEXT: v_and_b32_e32 v24, 0xffff, v24
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v22, v100, 16, v32
@@ -176312,7 +176322,7 @@ define inreg <64 x half> @bitcast_v128i8_to_v64f16_scalar(<128 x i8> inreg %a, i
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v6, 16, v65
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v49, 16, v48
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v10, 16, v50
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v51
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v52
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v15, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v16, 16, v38
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v19, v70, 16, v19
@@ -194353,7 +194363,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:404 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v0, off, s[0:3], s32 offset:220
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v31
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416 ; 4-byte Folded Spill
; VI-NEXT: buffer_load_ushort v34, off, s[0:3], s32 offset:216
@@ -195242,7 +195252,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX9-NEXT: buffer_load_ushort v17, off, s[0:3], s32 offset:20
; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 offset:16
; GFX9-NEXT: buffer_load_ushort v24, off, s[0:3], s32 offset:12
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_waitcnt vmcnt(41)
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424 ; 4-byte Folded Spill
; GFX9-NEXT: buffer_load_ushort v14, off, s[0:3], s32
@@ -196027,129 +196037,129 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:312
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:308
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:304
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:312
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:308
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:304
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v180, off, s32 offset:300
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:296
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:292
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:296
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:292
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v178, off, s32 offset:288
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v182, off, s32 offset:284
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:280
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:276
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:272
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:280
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:276
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:272
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v181, off, s32 offset:268
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v132, off, s32 offset:264
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v162, off, s32 offset:260
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:256
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:264
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:260
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:256
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v179, off, s32 offset:252
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:248
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v149, off, s32 offset:244
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:248
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:240
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v177, off, s32 offset:236
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:232
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:228
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v128, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:224
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v167, off, s32 offset:220
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:216
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v131, off, s32 offset:212
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v148, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:208
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v176, off, s32 offset:204
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:200
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:196
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v119, off, s32 offset:192
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v164, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v163, off, s32 offset:188
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:184
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v118, off, s32 offset:180
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:176
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v151, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v117, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v150, off, s32 offset:172
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v113, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v147, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v146, off, s32 offset:156
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v133, off, s32 offset:140
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v145, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v144, off, s32 offset:124
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v130, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v129, off, s32 offset:108
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:100
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v116, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:92
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:88
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:84
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v103, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:76
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:72
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:68
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:60
; GFX11-TRUE16-NEXT: s_clause 0xf
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v166, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v134, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v135, off, s32 offset:12
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v27
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v25
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v24
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v23
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v22
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v19
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v16
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v13
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0x6050400
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v70, v114, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v82, v114, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v32, v38, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v33, v50, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v11
@@ -196160,92 +196170,92 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s9, s4, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s9, s5, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s40, s8, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s14, s7, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s13, s8, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s4, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s12, s6, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s12, s7, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s5, s56, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s6, s56, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s61, s45, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s43, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s72, s45, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s42, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s60, s15, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, s13, s46, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s60, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, s14, s46, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s72, s58, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s57, s63, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s57, s62, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s59, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s59, s43, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s10, s41, v10
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, s47, s62, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s10, s40, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, s47, s61, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v37, v68, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s42, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v37, v69, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s73, s41, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v86, s74, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v53, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v134, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v135, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v66, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v39, v83, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v71, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v39, v166, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v70, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v99, v65, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v69, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v98, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v68, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v97, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v84, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v87, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v83, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v103, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v98, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v102, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v96, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v18.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v18, v116, v67, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v113, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v115, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v112, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v130, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v118, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v129, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v117, v84, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v20.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v145, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v64, v117, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v144, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v64, v116, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v21.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v135, v81, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v82, v131, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v133, v80, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v81, v130, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v22.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v147, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v87, v144, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v146, v99, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v85, v134, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v23.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v151, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v101, v149, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v150, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v100, v148, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v24.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, v119, v164, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v132, v162, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, v118, v163, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v131, v161, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v25.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v25, v176, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v30, v102, v160, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v176, v147, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v101, v151, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v26, v128, v167, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v129, v165, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v119, v167, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v128, v164, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v27.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v27, v177, v146, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v183, v133, v166, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v177, v145, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v183, v132, v165, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v28.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v28, v150, v179, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v28, v149, v179, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v29.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v29, v181, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v181, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v30.l
; GFX11-TRUE16-NEXT: v_perm_b32 v30, v178, v182, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v31.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v31, v180, v161, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v31, v180, v160, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v183.l
; GFX11-TRUE16-NEXT: s_branch .LBB97_3
; GFX11-TRUE16-NEXT: .LBB97_2:
@@ -196260,98 +196270,98 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v180
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v165
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v182
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v165
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v164
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v181
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v161, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v133, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v160, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v132, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v178, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v31, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v133, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v129, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v160
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v128, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v151
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v30, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v163, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v162, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v179
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v162
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v129, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v102, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v161
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v128, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v101, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v29, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v150, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v132, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v149, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v131, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v177
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v102, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v149
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v101, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v148
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v132, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v146, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v131, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v145, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v167
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v101, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v144
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v100, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v176
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v128, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v101, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v87, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v148, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v131
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v119, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v100, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v85, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v147, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v130
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v164
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v87, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v163
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v82, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v117
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v119, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v151
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v118
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v116
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v118, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v150
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v117
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v24, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v115, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v85, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v147
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v113, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v84, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v146
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v64, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v113
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v112
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v85, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v100, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v135
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v84, 0x300, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v99, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v133
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v98
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v145
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v96
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v144
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v81, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v80, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v112, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v84
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v103, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v83
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v130
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v129
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v80, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v116
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v71
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v68
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v115
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v70
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v49, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v67, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v103
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v102
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v97
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v87
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v99
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v98
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v48, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 0x300, v0
@@ -196359,81 +196369,82 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v83
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v166
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v134
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v135
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v114
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v2, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v96, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v97, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v68
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v70, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v69
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v82, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v86, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s42, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s62, s47, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s73, s41, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s61, s47, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s59, s44, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s59, s43, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-TRUE16-NEXT: s_add_i32 s12, s12, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s72, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s43, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s56, s5, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s42, s11, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s56, s6, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s12, s6, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s12, s7, v1
; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s41, s10, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s46, s13, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s40, s10, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s46, s14, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s60, s15, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s61, s45, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s60, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s45, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v65, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
; GFX11-TRUE16-NEXT: s_add_i32 s9, s9, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s63, s57, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v48, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s9, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s9, s5, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s40, s40, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s14, s7, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v54, s40, s8, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v54, s13, s8, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v67, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -196451,13 +196462,13 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v67, 0x300, v67
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v69, 0x300, v69
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v70
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v81, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v82, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v71
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v80
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v69.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v82.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v67.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v68.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v54.l
@@ -196478,15 +196489,15 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v49.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v52.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v84.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v87.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v101.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v132.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v102.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v129.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v133.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v81.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v100.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v28.h, v131.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v101.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v30.h, v128.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.h, v132.l
; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -196552,34 +196563,34 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v86, off, s32 offset:116
; GFX11-FAKE16-NEXT: scratch_load_u16 v83, off, s32 offset:112
; GFX11-FAKE16-NEXT: scratch_load_u16 v98, off, s32 offset:108
-; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:104
+; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:104
; GFX11-FAKE16-NEXT: scratch_load_u16 v71, off, s32 offset:100
; GFX11-FAKE16-NEXT: scratch_load_u16 v85, off, s32 offset:96
; GFX11-FAKE16-NEXT: scratch_load_u16 v96, off, s32 offset:92
-; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:88
+; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:88
; GFX11-FAKE16-NEXT: scratch_load_u16 v69, off, s32 offset:84
-; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:80
+; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:80
; GFX11-FAKE16-NEXT: scratch_load_u16 v82, off, s32 offset:76
-; GFX11-FAKE16-NEXT: scratch_load_u16 v55, off, s32 offset:72
-; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:68
-; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:64
+; GFX11-FAKE16-NEXT: scratch_load_u16 v54, off, s32 offset:72
+; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:68
+; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:64
; GFX11-FAKE16-NEXT: scratch_load_u16 v80, off, s32 offset:60
; GFX11-FAKE16-NEXT: s_clause 0xf
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:56
; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:52
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v65, off, s32 offset:44
+; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:48
+; GFX11-FAKE16-NEXT: scratch_load_u16 v64, off, s32 offset:44
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v31, off, s32 offset:316
+; GFX11-FAKE16-NEXT: scratch_load_u16 v68, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v67, off, s32 offset:12
+; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v66, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:8
-; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:4
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v30
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s61, v29
@@ -196594,7 +196605,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s46, v20
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s57, v19
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s60, v18
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v17
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v17
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s40, v16
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s47, v15
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s58, v14
@@ -196603,19 +196614,19 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s41, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s44, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s9, v9
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s12, v8
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v8
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s15, v7
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s42, v6
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v5
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v4
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s11, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s13, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0xc0c0004
@@ -196623,7 +196634,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v32, v48, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v36, v49, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v19, v68, v55, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v19, v67, v54, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v2, s16, s17, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s20, s21, v9
@@ -196639,7 +196650,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s11, s8, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s13, s8, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s28, s29, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v7, 16, v6
; GFX11-FAKE16-NEXT: v_perm_b32 v7, s42, s15, v9
@@ -196647,7 +196658,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: v_perm_b32 v6, s6, s4, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_perm_b32 v10, s12, s9, v9
+; GFX11-FAKE16-NEXT: v_perm_b32 v10, s11, s9, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
; GFX11-FAKE16-NEXT: v_perm_b32 v12, s44, s41, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v8, 16, v5
@@ -196658,7 +196669,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v7, s14, s10, v9
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v12
; GFX11-FAKE16-NEXT: v_perm_b32 v12, s60, s57, v9
-; GFX11-FAKE16-NEXT: v_perm_b32 v13, s13, s40, v11
+; GFX11-FAKE16-NEXT: v_perm_b32 v13, s12, s40, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX11-FAKE16-NEXT: v_perm_b32 v16, s45, s56, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v14
@@ -196669,8 +196680,8 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v13, s73, s72, v9
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v16, 16, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_perm_b32 v14, v67, v51, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v53, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v14, v66, v52, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v37, v68, 0x6050400
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v15, 16, v12
; GFX11-FAKE16-NEXT: v_perm_b32 v11, s61, s63, v11
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
@@ -196678,18 +196689,18 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xffff, v14
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v33, v38, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v33, v39, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff, v12
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v17, 16, v14
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v18, 16, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v80, v66, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v80, v65, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v15, 16, v12
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v65, v39, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v82, v64, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v64, v38, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v82, v55, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v17, v34, v50, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v69, v52, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v20, v69, v51, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v15, 0xffff, v15
; GFX11-FAKE16-NEXT: v_and_b32_e32 v18, 0xffff, v18
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v96, v85, 0xc0c0004
@@ -196697,7 +196708,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v98, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v17, 16, v15
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v20, 16, v18
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v71, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v71, v53, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v20, 0xffff, v21
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v86, v81, 0xc0c0004
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
@@ -196850,36 +196861,36 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v82
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v83, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v81, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v54, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v64, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v53, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v67
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v55, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v70, 0x300, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v80
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v65
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v64
; GFX11-FAKE16-NEXT: s_add_i32 s40, s40, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v52, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v66, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v55, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v39, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v51, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v65, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, v4, v54, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v38, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v67
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v66
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v54, 0x300, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v68
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v49
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v51, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v52, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v3, v3, v36, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v36, 0x300, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v38
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v34, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0xc0c0004
@@ -196894,19 +196905,19 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s62, s59, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s12, s12, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s11, s11, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s73, s72, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s40, s13, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s40, s12, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v32, 0x300, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s63, s61, v2
; GFX11-FAKE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-FAKE16-NEXT: s_add_i32 s60, s60, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v39, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s12, s9, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s11, s9, v2
; GFX11-FAKE16-NEXT: s_add_i32 s46, s46, 3
; GFX11-FAKE16-NEXT: s_add_i32 s58, s58, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s11, s11, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v33, 0x300, v3
@@ -196917,7 +196928,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s46, s43, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v5, s58, s47, v2
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s11, s8, v2
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s13, s8, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v50, s24, s25, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v65, s0, s1, v2
; GFX11-FAKE16-NEXT: s_add_i32 s44, s44, 3
@@ -196932,7 +196943,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s44, s41, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v3, s14, s10, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v4, s42, s15, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v51, 0x300, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v52, 0x300, v0
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v50, 0x300, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v67, s2, s3, v2
@@ -196959,7 +196970,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v5, s6, s4, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v53, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v55, 0x300, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
; GFX11-FAKE16-NEXT: v_perm_b32 v1, s20, s21, v2
; GFX11-FAKE16-NEXT: v_perm_b32 v66, s18, s19, v2
@@ -196975,7 +196986,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v35, 16, v33
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v13, v32, 16, v36
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v14, v34, 16, v14
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v54
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v53
; GFX11-FAKE16-NEXT: v_and_b32_e32 v33, 0xffff, v71
; GFX11-FAKE16-NEXT: v_and_b32_e32 v34, 0xffff, v18
; GFX11-FAKE16-NEXT: v_and_b32_e32 v35, 0xffff, v17
@@ -196986,9 +196997,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v64
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v68, 0x300, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v51, 0xffff, v51
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v55, 16, v32
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v52, 16, v33
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v52
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v17, v54, 16, v32
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v18, v51, 16, v33
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v20, v20, 16, v34
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v21, v21, 16, v35
; GFX11-FAKE16-NEXT: v_and_b32_e32 v32, 0xffff, v99
@@ -196999,9 +197010,9 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v66, 16, v64
; GFX11-FAKE16-NEXT: v_and_b32_e32 v64, 0xffff, v68
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v53, 16, v51
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v5, v55, 16, v52
; GFX11-FAKE16-NEXT: v_and_b32_e32 v48, 0xffff, v48
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v51, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v52, 0xffff, v7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v19, 0xffff, v19
; GFX11-FAKE16-NEXT: v_and_b32_e32 v24, 0xffff, v24
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v22, v100, 16, v32
@@ -197017,7 +197028,7 @@ define inreg <64 x i16> @bitcast_v128i8_to_v64i16_scalar(<128 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v6, 16, v65
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v7, v49, 16, v48
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v10, 16, v50
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v51
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v52
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v15, v15, 16, v37
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v16, v16, 16, v38
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v19, v70, 16, v19
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 280a0f17a4bc8a..7d34c764c45973 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -14357,7 +14357,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s76, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB27_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -14674,7 +14674,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB27_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -14935,9 +14935,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
@@ -14976,9 +14976,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -14986,7 +14986,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v52, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v38, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v39, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s22, s23, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v36, v32, 0xc0c0004
@@ -15038,7 +15038,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v13
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s8, s7, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -15166,7 +15166,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v14, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 3, v52
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 3, v48
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v37
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v36
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v33
@@ -15179,7 +15179,7 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v15, s4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v32, 0xc0c0004
@@ -15224,8 +15224,8 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -15265,9 +15265,9 @@ define inreg <16 x i32> @bitcast_v64i8_to_v16i32_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -29552,7 +29552,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; VI-NEXT: v_readfirstlane_b32 s76, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB51_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -29869,7 +29869,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB51_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -30130,9 +30130,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
@@ -30171,9 +30171,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -30181,7 +30181,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v52, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v38, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v39, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s22, s23, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v36, v32, 0xc0c0004
@@ -30233,7 +30233,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v13
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s8, s7, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -30361,7 +30361,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v14, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 3, v52
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 3, v48
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v37
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v36
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v33
@@ -30374,7 +30374,7 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-TRUE16-NEXT: v_perm_b32 v15, s4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v32, 0xc0c0004
@@ -30419,8 +30419,8 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -30460,9 +30460,9 @@ define inreg <16 x float> @bitcast_v64i8_to_v16f32_scalar(<64 x i8> inreg %a, i3
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -43625,7 +43625,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; VI-NEXT: v_readfirstlane_b32 s76, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB71_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -43942,7 +43942,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB71_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -44203,9 +44203,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
@@ -44244,9 +44244,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -44254,7 +44254,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v52, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v38, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v39, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s22, s23, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v36, v32, 0xc0c0004
@@ -44306,7 +44306,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v13
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s8, s7, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -44434,7 +44434,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v14, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 3, v52
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 3, v48
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v37
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v36
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v33
@@ -44447,7 +44447,7 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_perm_b32 v15, s4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v32, 0xc0c0004
@@ -44492,8 +44492,8 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -44533,9 +44533,9 @@ define inreg <8 x i64> @bitcast_v64i8_to_v8i64_scalar(<64 x i8> inreg %a, i32 in
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -57116,7 +57116,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s76, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB87_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -57433,7 +57433,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s76, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB87_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -57694,9 +57694,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:24
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:20
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:16
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
@@ -57735,9 +57735,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -57745,7 +57745,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v33, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v52, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v19, v38, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v39, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s22, s23, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v20, v36, v32, 0xc0c0004
@@ -57797,7 +57797,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v13
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v48, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, s8, s7, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -57925,7 +57925,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v14, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 3, v52
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 3, v48
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v37
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v36
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v33
@@ -57938,7 +57938,7 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v15, s4, v51, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v13, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v14, v14, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v16, v38, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v18, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v32, 0xc0c0004
@@ -57983,8 +57983,8 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:20
; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
@@ -58024,9 +58024,9 @@ define inreg <8 x double> @bitcast_v64i8_to_v8f64_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s73, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s74, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -71303,7 +71303,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s57, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB99_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -71626,7 +71626,7 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB99_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -71886,60 +71886,60 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v24
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v19
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v15
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0x6050400
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v48, v52, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v48, v51, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v31, v37, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v32, v36, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v11
@@ -71950,44 +71950,44 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s5, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s7, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s41, s4, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s13, s5, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s14, s6, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s9, s43, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s12, s42, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s8, s14, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s72, s46, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s8, s13, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, s12, s56, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, s11, s45, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s62, s58, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s62, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s43, s60, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s40, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s46, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s9, s47, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s57, s73, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v38, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v35, v39, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s63, s56, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v50, s74, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v49, s74, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v35, v49, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v34, v53, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v39, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v38, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX11-TRUE16-NEXT: s_branch .LBB99_3
; GFX11-TRUE16-NEXT: .LBB99_2:
@@ -72002,83 +72002,83 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v38
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v37
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v49
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v51
; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v5, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s63, s56, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s13, s8, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s42, s12, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s40, v1
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s47, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s45, s11, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s62, s58, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s46, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s6, v1
; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s60, s43, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s5, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s41, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, s15, s7, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v27, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -72125,18 +72125,18 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:56
-; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:52
+; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:44
+; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:44
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
-; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:12
+; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:20
+; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:8
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32
@@ -72172,16 +72172,16 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v52, v36, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v51, v36, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-FAKE16-NEXT: v_perm_b32 v17, v38, v51, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v17, v38, v50, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v33, v39, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v2, s16, s17, v9
@@ -72230,15 +72230,15 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v16, 16, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v34, s74, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v53, v48, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v35, v50, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v52, v49, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v35, v53, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v11, s59, s61, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v31, v37, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff, v12
; GFX11-FAKE16-NEXT: v_and_b32_e32 v15, 0xffff, v15
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v32, v49, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v20, v32, v48, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v18, 0xffff, v18
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v14, 16, v12
@@ -72256,19 +72256,19 @@ define inreg <32 x i16> @bitcast_v64i8_to_v32i16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v51
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v50
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v49
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v52
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v48
; GFX11-FAKE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v36, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 3, v51
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 3, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v35, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s44, s44, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v39
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v48, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v49, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
@@ -84040,7 +84040,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; VI-NEXT: v_readfirstlane_b32 s57, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB107_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -84363,7 +84363,7 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB107_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -84623,60 +84623,60 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v24
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v19
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v15
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0x6050400
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v48, v52, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v48, v51, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v31, v37, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v32, v36, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v11
@@ -84687,44 +84687,44 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s5, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s7, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s41, s4, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s13, s5, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s14, s6, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s9, s43, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s12, s42, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s8, s14, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s72, s46, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s8, s13, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, s12, s56, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, s11, s45, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s62, s58, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s62, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s43, s60, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s40, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s46, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s9, s47, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s57, s73, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v38, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v35, v39, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s63, s56, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v50, s74, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v49, s74, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v35, v49, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v34, v53, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v39, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v38, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX11-TRUE16-NEXT: s_branch .LBB107_3
; GFX11-TRUE16-NEXT: .LBB107_2:
@@ -84739,83 +84739,83 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v38
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v37
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v49
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v51
; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v5, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s63, s56, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s13, s8, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s42, s12, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s40, v1
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s47, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s45, s11, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s62, s58, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s46, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s6, v1
; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s60, s43, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s5, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s41, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, s15, s7, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v27, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -84862,18 +84862,18 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:56
-; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:52
+; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:44
+; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:44
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
-; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:12
+; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:20
+; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:8
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32
@@ -84909,16 +84909,16 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v52, v36, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v51, v36, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-FAKE16-NEXT: v_perm_b32 v17, v38, v51, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v17, v38, v50, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v33, v39, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v2, s16, s17, v9
@@ -84967,15 +84967,15 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v16, 16, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v34, s74, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v53, v48, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v35, v50, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v52, v49, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v35, v53, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v11, s59, s61, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v31, v37, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff, v12
; GFX11-FAKE16-NEXT: v_and_b32_e32 v15, 0xffff, v15
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v32, v49, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v20, v32, v48, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v18, 0xffff, v18
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v14, 16, v12
@@ -84993,19 +84993,19 @@ define inreg <32 x half> @bitcast_v64i8_to_v32f16_scalar(<64 x i8> inreg %a, i32
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v51
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v50
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v49
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v52
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v48
; GFX11-FAKE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v36, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 3, v51
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 3, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v35, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s44, s44, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v39
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v48, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v49, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
@@ -94679,7 +94679,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; VI-NEXT: v_readfirstlane_b32 s57, v0
; VI-NEXT: s_waitcnt vmcnt(14)
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; VI-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cbranch_scc0 .LBB111_2
; VI-NEXT: ; %bb.1: ; %cmp.false
; VI-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -95002,7 +95002,7 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX9-NEXT: v_readfirstlane_b32 s63, v0
; GFX9-NEXT: s_waitcnt vmcnt(19)
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v35
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB111_2
; GFX9-NEXT: ; %bb.1: ; %cmp.false
; GFX9-NEXT: v_mov_b32_e32 v11, 0xc0c0004
@@ -95262,60 +95262,60 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:56
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:52
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:44
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:40
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:32
; GFX11-TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:28
; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s74, v30
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v29
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v28
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v27
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v26
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v25
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v24
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v29
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s47, v28
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s56, v27
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v26
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s11, v25
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v24
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s57, v23
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s73, v22
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v21
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v19
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v20
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v19
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s61, v18
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s9, v17
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v16
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v15
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v14
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s13, v12
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s12, v17
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s42, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s43, v15
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v14
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s14, v12
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s58, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s63, v10
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v9
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s40, v7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s62, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s44, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s59, v6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s41, v4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s45, v3
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s60, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s15, v4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s46, v3
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s72, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s75, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, 0x6050400
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v48, v52, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v48, v51, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v15, v31, v37, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v32, v36, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s2, s3, v11
@@ -95326,44 +95326,44 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_perm_b32 v2, s20, s21, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s4, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s10, s5, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v3, s24, s25, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s41, s7, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s15, s6, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s15, s7, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s41, s4, v11
; GFX11-TRUE16-NEXT: v_perm_b32 v1, s16, s17, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s13, s5, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s14, s6, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s28, s29, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s9, s43, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s12, s42, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s60, s45, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s8, s14, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s72, s46, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s8, s13, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s40, v11
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, s12, s56, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, s59, s44, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, s11, s45, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, s63, s58, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, s62, s58, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, s44, s62, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, s43, s60, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s42, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, s61, s40, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v12.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, s11, s46, v10
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, s9, s47, v10
; GFX11-TRUE16-NEXT: v_perm_b32 v10, s57, s73, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v13.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v38, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, s72, s47, v11
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v35, v39, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, s63, s56, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v12.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v50, s74, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v49, s74, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v53, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v14.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v35, v49, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v34, v53, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v39, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v38, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v16.l
; GFX11-TRUE16-NEXT: s_branch .LBB111_3
; GFX11-TRUE16-NEXT: .LBB111_2:
@@ -95378,83 +95378,83 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v36
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v39
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v38
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v37
; GFX11-TRUE16-NEXT: s_add_i32 s74, s74, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v49
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v51
; GFX11-TRUE16-NEXT: s_add_i32 s61, s61, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: s_add_i32 s73, s73, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s63, s63, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v2
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v5, v48, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s43, s43, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s42, s42, 3
; GFX11-TRUE16-NEXT: v_perm_b32 v4, s73, s57, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v38
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v39
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s72, s47, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s13, s13, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s63, s56, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s14, s14, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 0x300, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s74, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s14, s8, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s43, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s13, s8, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s42, s12, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_add_i32 s46, s46, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s56, s56, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_add_i32 s47, s47, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s45, s45, 3
; GFX11-TRUE16-NEXT: s_add_i32 s59, s59, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s72, s72, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s42, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s61, s40, v1
; GFX11-TRUE16-NEXT: s_add_i32 s28, s28, 3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s46, s11, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, s56, s12, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s47, s9, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, s45, s11, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s63, s58, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s62, s58, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s40, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, s60, s45, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s62, s62, 3
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s59, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, s72, s46, v1
+; GFX11-TRUE16-NEXT: s_add_i32 s60, s60, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, s13, s5, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, s14, s6, v1
; GFX11-TRUE16-NEXT: s_add_i32 s10, s10, 3
; GFX11-TRUE16-NEXT: s_add_i32 s20, s20, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s62, s44, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s60, s43, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s28, s29, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v4
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, s10, s5, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, s20, s21, v1
-; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s41, s41, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s15, s15, 3
; GFX11-TRUE16-NEXT: s_add_i32 s26, s26, 3
; GFX11-TRUE16-NEXT: s_add_i32 s22, s22, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_add_i32 s18, s18, 3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v3
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, s15, s6, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v24, s41, s7, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, s41, s4, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v24, s15, s7, v1
; GFX11-TRUE16-NEXT: v_perm_b32 v27, s26, s27, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v28, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v0
@@ -95501,18 +95501,18 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_clause 0xf
; GFX11-FAKE16-NEXT: scratch_load_u16 v32, off, s32 offset:56
-; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:52
+; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:52
; GFX11-FAKE16-NEXT: scratch_load_u16 v36, off, s32 offset:48
-; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:44
+; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:44
; GFX11-FAKE16-NEXT: scratch_load_u16 v33, off, s32 offset:40
; GFX11-FAKE16-NEXT: scratch_load_u16 v39, off, s32 offset:36
; GFX11-FAKE16-NEXT: scratch_load_u16 v35, off, s32 offset:32
-; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_b32 v54, off, s32 offset:60
+; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:28
; GFX11-FAKE16-NEXT: scratch_load_u16 v38, off, s32 offset:24
-; GFX11-FAKE16-NEXT: scratch_load_u16 v51, off, s32 offset:20
-; GFX11-FAKE16-NEXT: scratch_load_u16 v48, off, s32 offset:16
-; GFX11-FAKE16-NEXT: scratch_load_u16 v53, off, s32 offset:12
+; GFX11-FAKE16-NEXT: scratch_load_u16 v50, off, s32 offset:20
+; GFX11-FAKE16-NEXT: scratch_load_u16 v49, off, s32 offset:16
+; GFX11-FAKE16-NEXT: scratch_load_u16 v52, off, s32 offset:12
; GFX11-FAKE16-NEXT: scratch_load_u16 v31, off, s32 offset:8
; GFX11-FAKE16-NEXT: scratch_load_u16 v37, off, s32 offset:4
; GFX11-FAKE16-NEXT: scratch_load_u16 v34, off, s32
@@ -95548,16 +95548,16 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s75, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(8)
; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v54
-; GFX11-FAKE16-NEXT: s_and_b32 s76, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v9, 0xc0c0004
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v11, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v18, v52, v36, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v18, v51, v36, 0xc0c0004
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-FAKE16-NEXT: v_perm_b32 v17, v38, v51, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v17, v38, v50, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v33, v39, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v2, s16, s17, v9
@@ -95606,15 +95606,15 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v16, 16, v14
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_perm_b32 v12, v34, s74, 0x6050400
-; GFX11-FAKE16-NEXT: v_perm_b32 v15, v53, v48, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v16, v35, v50, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v15, v52, v49, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v16, v35, v53, 0x6050400
; GFX11-FAKE16-NEXT: v_perm_b32 v11, s59, s61, v11
; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xffff, v13
; GFX11-FAKE16-NEXT: v_perm_b32 v14, v31, v37, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v12, 0xffff, v12
; GFX11-FAKE16-NEXT: v_and_b32_e32 v15, 0xffff, v15
; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX11-FAKE16-NEXT: v_perm_b32 v20, v32, v49, 0x6050400
+; GFX11-FAKE16-NEXT: v_perm_b32 v20, v32, v48, 0x6050400
; GFX11-FAKE16-NEXT: v_and_b32_e32 v18, 0xffff, v18
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v11, 16, v13
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v12, v14, 16, v12
@@ -95632,19 +95632,19 @@ define inreg <32 x bfloat> @bitcast_v64i8_to_v32bf16_scalar(<64 x i8> inreg %a,
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s75, 1
; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v52
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 3, v51
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v53
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v50
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v49
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 3, v52
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 3, v48
; GFX11-FAKE16-NEXT: s_add_i32 s56, s56, 3
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v36, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 3, v51
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 3, v50
; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v35, 0xc0c0004
; GFX11-FAKE16-NEXT: s_add_i32 s44, s44, 3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 3, v39
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v48, 0xc0c0004
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v4, v49, 0xc0c0004
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0xc0c0004
; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
diff --git a/llvm/test/CodeGen/AMDGPU/anyext.ll b/llvm/test/CodeGen/AMDGPU/anyext.ll
index 29b45774f395ef..8cd5209f633db3 100644
--- a/llvm/test/CodeGen/AMDGPU/anyext.ll
+++ b/llvm/test/CodeGen/AMDGPU/anyext.ll
@@ -163,7 +163,7 @@ define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX8-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX8-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8-NEXT: s_cselect_b32 s0, 1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -179,7 +179,7 @@ define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
; GFX9-NEXT: v_and_b32_e32 v0, 0x80018001, v0
; GFX9-NEXT: v_bfi_b32 v0, v1, 0, v0
; GFX9-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index c424df950c5c0b..284814e32f39bc 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -13509,19 +13509,19 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX8_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[4:5]
-; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -13564,19 +13564,19 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX9_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[4:5]
-; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -13624,7 +13624,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -13675,7 +13675,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[6:7]
-; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -13728,7 +13728,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -13783,7 +13783,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[6:7]
-; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -15714,19 +15714,19 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX8_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
-; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -15769,19 +15769,19 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX9_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
-; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -15829,7 +15829,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -15880,7 +15880,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[6:7]
-; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -15933,7 +15933,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -15988,7 +15988,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[6:7]
-; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -17912,19 +17912,19 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX8_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
-; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -17966,19 +17966,19 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX9_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
-; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -18025,7 +18025,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -18075,7 +18075,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[6:7]
-; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -18128,7 +18128,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -18182,7 +18182,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[6:7]
-; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -20100,19 +20100,19 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX8_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[4:5]
-; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -20154,19 +20154,19 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s7
; GFX9_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[4:5]
-; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
-; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s6
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s8
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -20213,7 +20213,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -20263,7 +20263,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[6:7]
-; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
@@ -20316,7 +20316,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s[8:9], s[0:1], s[6:7]
-; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
@@ -20370,7 +20370,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[6:7]
-; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s8, 0
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 43100e18394266..0450b6cf7401ad 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -91,7 +91,7 @@ define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {
; GFX-942-NEXT: s_or_b32 s4, s1, 0x400000
; GFX-942-NEXT: s_add_i32 s5, s2, 0x7fff
; GFX-942-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s1
-; GFX-942-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX-942-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX-942-NEXT: s_cselect_b32 s1, s4, s5
; GFX-942-NEXT: s_lshr_b32 s2, s1, 16
; GFX-942-NEXT: s_bfe_u32 s1, s0, 0x10010
@@ -99,7 +99,7 @@ define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {
; GFX-942-NEXT: s_or_b32 s3, s0, 0x400000
; GFX-942-NEXT: s_add_i32 s4, s1, 0x7fff
; GFX-942-NEXT: v_cmp_u_f32_e64 s[0:1], s0, s0
-; GFX-942-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX-942-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX-942-NEXT: s_cselect_b32 s0, s3, s4
; GFX-942-NEXT: s_lshr_b32 s0, s0, 16
; GFX-942-NEXT: s_pack_ll_b32_b16 s0, s0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg-scc-clobber.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg-scc-clobber.ll
index 122c0f7068f4fb..b8408f3a686dd2 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-to-reg-scc-clobber.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg-scc-clobber.ll
@@ -15,13 +15,13 @@ define protected amdgpu_kernel void @sccClobber(ptr addrspace(1) %a, ptr addrspa
; RRLIST-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x44
; RRLIST-NEXT: s_load_dword s17, s[14:15], 0x0
; RRLIST-NEXT: s_waitcnt lgkmcnt(0)
-; RRLIST-NEXT: s_min_i32 s8, s16, 0
+; RRLIST-NEXT: s_min_i32 s4, s16, 0
; RRLIST-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; RRLIST-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
-; RRLIST-NEXT: s_and_b64 s[4:5], vcc, exec
-; RRLIST-NEXT: s_cselect_b32 s4, s16, s17
+; RRLIST-NEXT: s_cmp_lg_u64 vcc, 0
+; RRLIST-NEXT: s_cselect_b32 s5, s16, s17
; RRLIST-NEXT: s_cmp_eq_u64 s[2:3], s[0:1]
-; RRLIST-NEXT: s_cselect_b32 s0, s8, s4
+; RRLIST-NEXT: s_cselect_b32 s0, s4, s5
; RRLIST-NEXT: v_mov_b32_e32 v0, s0
; RRLIST-NEXT: global_store_dword v2, v0, s[6:7]
; RRLIST-NEXT: s_endpgm
@@ -37,13 +37,13 @@ define protected amdgpu_kernel void @sccClobber(ptr addrspace(1) %a, ptr addrspa
; FAST-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x44
; FAST-NEXT: s_load_dword s17, s[14:15], 0x0
; FAST-NEXT: s_waitcnt lgkmcnt(0)
-; FAST-NEXT: s_min_i32 s8, s16, 0
+; FAST-NEXT: s_min_i32 s4, s16, 0
; FAST-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; FAST-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
-; FAST-NEXT: s_and_b64 s[4:5], vcc, exec
-; FAST-NEXT: s_cselect_b32 s4, s16, s17
+; FAST-NEXT: s_cmp_lg_u64 vcc, 0
+; FAST-NEXT: s_cselect_b32 s5, s16, s17
; FAST-NEXT: s_cmp_eq_u64 s[2:3], s[0:1]
-; FAST-NEXT: s_cselect_b32 s0, s8, s4
+; FAST-NEXT: s_cselect_b32 s0, s4, s5
; FAST-NEXT: v_mov_b32_e32 v0, s0
; FAST-NEXT: global_store_dword v2, v0, s[6:7]
; FAST-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index 176dffac4abc5a..eecc9543ca9594 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -1674,7 +1674,7 @@ define amdgpu_gfx <8 x i16> @vec_16xi16_extract_8xi16_0(i1 inreg %cond, ptr addr
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v4, 1, v4
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT: s_and_b64 s[34:35], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB7_2
; GFX9-NEXT: ; %bb.1: ; %F
; GFX9-NEXT: global_load_dwordx4 v[4:7], v[2:3], off offset:16 glc
@@ -1735,7 +1735,7 @@ define amdgpu_gfx <8 x i16> @vec_16xi16_extract_8xi16_0(i1 inreg %cond, ptr addr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 1, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB7_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %F
; GFX11-TRUE16-NEXT: global_load_b128 v[4:7], v[2:3], off offset:16 glc dlc
@@ -1790,7 +1790,7 @@ define amdgpu_gfx <8 x i16> @vec_16xi16_extract_8xi16_0(i1 inreg %cond, ptr addr
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 1, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB7_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %F
; GFX11-FAKE16-NEXT: global_load_b128 v[4:7], v[2:3], off offset:16 glc dlc
@@ -2018,7 +2018,7 @@ define amdgpu_gfx <8 x half> @vec_16xf16_extract_8xf16_0(i1 inreg %cond, ptr add
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v4, 1, v4
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX9-NEXT: s_and_b64 s[34:35], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB8_2
; GFX9-NEXT: ; %bb.1: ; %F
; GFX9-NEXT: global_load_dwordx4 v[4:7], v[2:3], off offset:16 glc
@@ -2077,7 +2077,7 @@ define amdgpu_gfx <8 x half> @vec_16xf16_extract_8xf16_0(i1 inreg %cond, ptr add
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 1, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB8_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %F
; GFX11-TRUE16-NEXT: global_load_b128 v[4:7], v[2:3], off offset:16 glc dlc
@@ -2132,7 +2132,7 @@ define amdgpu_gfx <8 x half> @vec_16xf16_extract_8xf16_0(i1 inreg %cond, ptr add
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 1, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB8_2
; GFX11-FAKE16-NEXT: ; %bb.1: ; %F
; GFX11-FAKE16-NEXT: global_load_b128 v[4:7], v[2:3], off offset:16 glc dlc
diff --git a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
index d2c4f31d816ecc..46b52832998926 100644
--- a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
@@ -55,7 +55,7 @@ define amdgpu_kernel void @fcmp_f16_lt(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_lt_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -82,7 +82,7 @@ define amdgpu_kernel void @fcmp_f16_lt(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -110,7 +110,7 @@ define amdgpu_kernel void @fcmp_f16_lt(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -227,7 +227,7 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_lt_f16_e64 s[0:1], |v0|, |v1|
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -254,7 +254,7 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_lt_f16_e64 s2, |v0.l|, |v0.h|
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -282,7 +282,7 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_lt_f16_e64 s2, |v0|, |v1|
-; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -407,7 +407,7 @@ define amdgpu_kernel void @fcmp_f16_eq(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_eq_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -434,7 +434,7 @@ define amdgpu_kernel void @fcmp_f16_eq(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -462,7 +462,7 @@ define amdgpu_kernel void @fcmp_f16_eq(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -579,7 +579,7 @@ define amdgpu_kernel void @fcmp_f16_le(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_le_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -606,7 +606,7 @@ define amdgpu_kernel void @fcmp_f16_le(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -634,7 +634,7 @@ define amdgpu_kernel void @fcmp_f16_le(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -751,7 +751,7 @@ define amdgpu_kernel void @fcmp_f16_gt(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_gt_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -778,7 +778,7 @@ define amdgpu_kernel void @fcmp_f16_gt(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -806,7 +806,7 @@ define amdgpu_kernel void @fcmp_f16_gt(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -923,7 +923,7 @@ define amdgpu_kernel void @fcmp_f16_lg(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_lg_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -950,7 +950,7 @@ define amdgpu_kernel void @fcmp_f16_lg(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -978,7 +978,7 @@ define amdgpu_kernel void @fcmp_f16_lg(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1095,7 +1095,7 @@ define amdgpu_kernel void @fcmp_f16_ge(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_ge_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -1122,7 +1122,7 @@ define amdgpu_kernel void @fcmp_f16_ge(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1150,7 +1150,7 @@ define amdgpu_kernel void @fcmp_f16_ge(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1267,7 +1267,7 @@ define amdgpu_kernel void @fcmp_f16_o(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_o_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -1294,7 +1294,7 @@ define amdgpu_kernel void @fcmp_f16_o(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1322,7 +1322,7 @@ define amdgpu_kernel void @fcmp_f16_o(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1439,7 +1439,7 @@ define amdgpu_kernel void @fcmp_f16_u(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_u_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -1466,7 +1466,7 @@ define amdgpu_kernel void @fcmp_f16_u(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1494,7 +1494,7 @@ define amdgpu_kernel void @fcmp_f16_u(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1611,7 +1611,7 @@ define amdgpu_kernel void @fcmp_f16_nge(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_nge_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -1638,7 +1638,7 @@ define amdgpu_kernel void @fcmp_f16_nge(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1666,7 +1666,7 @@ define amdgpu_kernel void @fcmp_f16_nge(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1783,7 +1783,7 @@ define amdgpu_kernel void @fcmp_f16_nlg(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_nlg_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -1810,7 +1810,7 @@ define amdgpu_kernel void @fcmp_f16_nlg(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1838,7 +1838,7 @@ define amdgpu_kernel void @fcmp_f16_nlg(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -1955,7 +1955,7 @@ define amdgpu_kernel void @fcmp_f16_ngt(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_ngt_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -1982,7 +1982,7 @@ define amdgpu_kernel void @fcmp_f16_ngt(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2010,7 +2010,7 @@ define amdgpu_kernel void @fcmp_f16_ngt(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2127,7 +2127,7 @@ define amdgpu_kernel void @fcmp_f16_nle(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_nle_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -2154,7 +2154,7 @@ define amdgpu_kernel void @fcmp_f16_nle(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2182,7 +2182,7 @@ define amdgpu_kernel void @fcmp_f16_nle(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2299,7 +2299,7 @@ define amdgpu_kernel void @fcmp_f16_neq(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_neq_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -2326,7 +2326,7 @@ define amdgpu_kernel void @fcmp_f16_neq(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2354,7 +2354,7 @@ define amdgpu_kernel void @fcmp_f16_neq(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2471,7 +2471,7 @@ define amdgpu_kernel void @fcmp_f16_nlt(
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cmp_nlt_f16_e32 vcc, v0, v1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -2498,7 +2498,7 @@ define amdgpu_kernel void @fcmp_f16_nlt(
; GFX11-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s1
; GFX11-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2526,7 +2526,7 @@ define amdgpu_kernel void @fcmp_f16_nlt(
; GFX11-FAKE16-NEXT: s_mov_b32 s8, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s9, s1
; GFX11-FAKE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0, v1
-; GFX11-FAKE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -2654,9 +2654,9 @@ define amdgpu_kernel void @fcmp_v2f16_lt(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_lt_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_lt_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -2688,9 +2688,9 @@ define amdgpu_kernel void @fcmp_v2f16_lt(
; GFX11-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -2721,9 +2721,9 @@ define amdgpu_kernel void @fcmp_v2f16_lt(
; GFX11-FAKE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_lt_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -2863,9 +2863,9 @@ define amdgpu_kernel void @fcmp_v2f16_eq(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_eq_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_eq_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -2897,9 +2897,9 @@ define amdgpu_kernel void @fcmp_v2f16_eq(
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -2930,9 +2930,9 @@ define amdgpu_kernel void @fcmp_v2f16_eq(
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3071,9 +3071,9 @@ define amdgpu_kernel void @fcmp_v2f16_le(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_le_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_le_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -3105,9 +3105,9 @@ define amdgpu_kernel void @fcmp_v2f16_le(
; GFX11-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_le_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3138,9 +3138,9 @@ define amdgpu_kernel void @fcmp_v2f16_le(
; GFX11-FAKE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_le_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3279,9 +3279,9 @@ define amdgpu_kernel void @fcmp_v2f16_gt(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_gt_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_gt_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -3313,9 +3313,9 @@ define amdgpu_kernel void @fcmp_v2f16_gt(
; GFX11-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3346,9 +3346,9 @@ define amdgpu_kernel void @fcmp_v2f16_gt(
; GFX11-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3488,9 +3488,9 @@ define amdgpu_kernel void @fcmp_v2f16_lg(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_lg_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_lg_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -3522,9 +3522,9 @@ define amdgpu_kernel void @fcmp_v2f16_lg(
; GFX11-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_lg_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3555,9 +3555,9 @@ define amdgpu_kernel void @fcmp_v2f16_lg(
; GFX11-FAKE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_lg_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3697,9 +3697,9 @@ define amdgpu_kernel void @fcmp_v2f16_ge(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_ge_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_ge_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -3731,9 +3731,9 @@ define amdgpu_kernel void @fcmp_v2f16_ge(
; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3764,9 +3764,9 @@ define amdgpu_kernel void @fcmp_v2f16_ge(
; GFX11-FAKE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_ge_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3906,9 +3906,9 @@ define amdgpu_kernel void @fcmp_v2f16_o(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_o_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_o_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -3940,9 +3940,9 @@ define amdgpu_kernel void @fcmp_v2f16_o(
; GFX11-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_o_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -3973,9 +3973,9 @@ define amdgpu_kernel void @fcmp_v2f16_o(
; GFX11-FAKE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_o_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4115,9 +4115,9 @@ define amdgpu_kernel void @fcmp_v2f16_u(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_u_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_u_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -4149,9 +4149,9 @@ define amdgpu_kernel void @fcmp_v2f16_u(
; GFX11-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_u_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4182,9 +4182,9 @@ define amdgpu_kernel void @fcmp_v2f16_u(
; GFX11-FAKE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_u_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4323,9 +4323,9 @@ define amdgpu_kernel void @fcmp_v2f16_nge(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_nge_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_nge_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -4357,9 +4357,9 @@ define amdgpu_kernel void @fcmp_v2f16_nge(
; GFX11-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_nge_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4390,9 +4390,9 @@ define amdgpu_kernel void @fcmp_v2f16_nge(
; GFX11-FAKE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_nge_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4531,9 +4531,9 @@ define amdgpu_kernel void @fcmp_v2f16_nlg(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_nlg_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_nlg_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -4565,9 +4565,9 @@ define amdgpu_kernel void @fcmp_v2f16_nlg(
; GFX11-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_nlg_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4598,9 +4598,9 @@ define amdgpu_kernel void @fcmp_v2f16_nlg(
; GFX11-FAKE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_nlg_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4740,9 +4740,9 @@ define amdgpu_kernel void @fcmp_v2f16_ngt(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_ngt_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_ngt_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -4774,9 +4774,9 @@ define amdgpu_kernel void @fcmp_v2f16_ngt(
; GFX11-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_ngt_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4807,9 +4807,9 @@ define amdgpu_kernel void @fcmp_v2f16_ngt(
; GFX11-FAKE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_ngt_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -4948,9 +4948,9 @@ define amdgpu_kernel void @fcmp_v2f16_nle(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_nle_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_nle_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -4982,9 +4982,9 @@ define amdgpu_kernel void @fcmp_v2f16_nle(
; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_nle_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -5015,9 +5015,9 @@ define amdgpu_kernel void @fcmp_v2f16_nle(
; GFX11-FAKE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_nle_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -5156,9 +5156,9 @@ define amdgpu_kernel void @fcmp_v2f16_neq(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_neq_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_neq_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -5190,9 +5190,9 @@ define amdgpu_kernel void @fcmp_v2f16_neq(
; GFX11-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_neq_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -5223,9 +5223,9 @@ define amdgpu_kernel void @fcmp_v2f16_neq(
; GFX11-FAKE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_neq_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -5364,9 +5364,9 @@ define amdgpu_kernel void @fcmp_v2f16_nlt(
; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-NEXT: v_cmp_nlt_f16_e32 vcc, v1, v0
; VI-NEXT: v_cmp_nlt_f16_e64 s[0:1], v3, v2
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, -1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, -1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s0
@@ -5398,9 +5398,9 @@ define amdgpu_kernel void @fcmp_v2f16_nlt(
; GFX11-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v1.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_nlt_f16_e64 s0, v3.l, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
@@ -5431,9 +5431,9 @@ define amdgpu_kernel void @fcmp_v2f16_nlt(
; GFX11-FAKE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_cmp_nlt_f16_e64 s0, v3, v2
-; GFX11-FAKE16-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
; GFX11-FAKE16-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
index 04632801c6ed14..83f2ba1fc57cff 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
@@ -3091,7 +3091,7 @@ define amdgpu_ps i16 @s_copysign_out_bf16_mag_f32_sign_bf16(float inreg %mag, bf
; GFX8-NEXT: s_or_b32 s4, s0, 0x400000
; GFX8-NEXT: s_add_i32 s6, s2, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[2:3], s0, s0
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cselect_b32 s0, s4, s6
; GFX8-NEXT: s_lshr_b32 s0, s0, 16
; GFX8-NEXT: s_movk_i32 s5, 0x7fff
@@ -3108,7 +3108,7 @@ define amdgpu_ps i16 @s_copysign_out_bf16_mag_f32_sign_bf16(float inreg %mag, bf
; GFX9-NEXT: s_or_b32 s4, s0, 0x400000
; GFX9-NEXT: s_add_i32 s6, s2, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[2:3], s0, s0
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cselect_b32 s0, s4, s6
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: s_movk_i32 s5, 0x7fff
@@ -3126,7 +3126,7 @@ define amdgpu_ps i16 @s_copysign_out_bf16_mag_f32_sign_bf16(float inreg %mag, bf
; GFX10-NEXT: s_bitset1_b32 s0, 22
; GFX10-NEXT: s_addk_i32 s2, 0x7fff
; GFX10-NEXT: v_mov_b32_e32 v0, s1
-; GFX10-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
; GFX10-NEXT: s_cselect_b32 s0, s0, s2
; GFX10-NEXT: s_lshr_b32 s0, s0, 16
; GFX10-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
@@ -3141,7 +3141,7 @@ define amdgpu_ps i16 @s_copysign_out_bf16_mag_f32_sign_bf16(float inreg %mag, bf
; GFX11TRUE16-NEXT: s_bitset1_b32 s0, 22
; GFX11TRUE16-NEXT: s_addk_i32 s2, 0x7fff
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
-; GFX11TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11TRUE16-NEXT: s_cselect_b32 s0, s0, s2
; GFX11TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11TRUE16-NEXT: s_lshr_b32 s0, s0, 16
@@ -3159,7 +3159,7 @@ define amdgpu_ps i16 @s_copysign_out_bf16_mag_f32_sign_bf16(float inreg %mag, bf
; GFX11FAKE16-NEXT: s_bitset1_b32 s0, 22
; GFX11FAKE16-NEXT: s_addk_i32 s2, 0x7fff
; GFX11FAKE16-NEXT: v_mov_b32_e32 v0, s1
-; GFX11FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11FAKE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11FAKE16-NEXT: s_cselect_b32 s0, s0, s2
; GFX11FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11FAKE16-NEXT: s_lshr_b32 s0, s0, 16
@@ -4326,14 +4326,14 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f32_sign_v2bf16(<2 x float> in
; GFX8-NEXT: s_or_b32 s3, s0, 0x400000
; GFX8-NEXT: s_add_i32 s6, s4, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], s0, s0
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8-NEXT: s_cselect_b32 s0, s3, s6
; GFX8-NEXT: s_bfe_u32 s4, s1, 0x10010
; GFX8-NEXT: s_add_i32 s4, s4, s1
; GFX8-NEXT: s_or_b32 s3, s1, 0x400000
; GFX8-NEXT: s_add_i32 s6, s4, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], s1, s1
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8-NEXT: s_cselect_b32 s1, s3, s6
; GFX8-NEXT: s_lshr_b32 s1, s1, 16
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], 16
@@ -4351,7 +4351,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f32_sign_v2bf16(<2 x float> in
; GFX9-NEXT: s_or_b32 s3, s1, 0x400000
; GFX9-NEXT: s_add_i32 s6, s4, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[4:5], s1, s1
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cselect_b32 s1, s3, s6
; GFX9-NEXT: s_lshr_b32 s3, s1, 16
; GFX9-NEXT: s_bfe_u32 s1, s0, 0x10010
@@ -4359,7 +4359,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f32_sign_v2bf16(<2 x float> in
; GFX9-NEXT: s_or_b32 s4, s0, 0x400000
; GFX9-NEXT: s_add_i32 s5, s1, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[0:1], s0, s0
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9-NEXT: s_cselect_b32 s0, s4, s5
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s3
@@ -4378,15 +4378,15 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f32_sign_v2bf16(<2 x float> in
; GFX10-NEXT: s_bitset1_b32 s1, 22
; GFX10-NEXT: s_addk_i32 s3, 0x7fff
; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, s0, s0
; GFX10-NEXT: s_cselect_b32 s1, s1, s3
; GFX10-NEXT: s_bfe_u32 s3, s0, 0x10010
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, s0, s0
-; GFX10-NEXT: s_add_i32 s3, s3, s0
; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_add_i32 s3, s3, s0
; GFX10-NEXT: s_bitset1_b32 s0, 22
; GFX10-NEXT: s_addk_i32 s3, 0x7fff
-; GFX10-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: s_cselect_b32 s0, s0, s3
; GFX10-NEXT: s_lshr_b32 s0, s0, 16
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
@@ -4402,15 +4402,15 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f32_sign_v2bf16(<2 x float> in
; GFX11-NEXT: s_bitset1_b32 s1, 22
; GFX11-NEXT: s_addk_i32 s3, 0x7fff
; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: v_cmp_u_f32_e64 s4, s0, s0
; GFX11-NEXT: s_cselect_b32 s1, s1, s3
; GFX11-NEXT: s_bfe_u32 s3, s0, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s4, s0, s0
-; GFX11-NEXT: s_add_i32 s3, s3, s0
; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_add_i32 s3, s3, s0
; GFX11-NEXT: s_bitset1_b32 s0, 22
; GFX11-NEXT: s_addk_i32 s3, 0x7fff
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_cselect_b32 s0, s0, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s0, s0, 16
@@ -4473,7 +4473,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX8-NEXT: v_cmp_nlg_f64_e64 s[0:1], s[2:3], v[0:1]
; GFX8-NEXT: v_cmp_gt_f64_e64 s[12:13], |s[2:3]|, |v[0:1]|
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX8-NEXT: s_cselect_b32 s8, 1, -1
; GFX8-NEXT: s_add_i32 s14, s5, s8
; GFX8-NEXT: s_and_b64 s[8:9], s[10:11], exec
@@ -4482,13 +4482,13 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX8-NEXT: s_add_i32 s8, s8, s5
; GFX8-NEXT: s_addk_i32 s8, 0x7fff
; GFX8-NEXT: s_bitset1_b32 s5, 22
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX8-NEXT: s_cselect_b32 s6, s5, s8
; GFX8-NEXT: v_readfirstlane_b32 s5, v3
; GFX8-NEXT: s_bitcmp1_b32 s5, 0
; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[8:9]
-; GFX8-NEXT: s_and_b64 s[8:9], s[12:13], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[12:13], 0
; GFX8-NEXT: v_cmp_u_f64_e64 s[2:3], s[2:3], s[2:3]
; GFX8-NEXT: s_cselect_b32 s7, 1, -1
; GFX8-NEXT: s_add_i32 s7, s5, s7
@@ -4496,10 +4496,10 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX8-NEXT: s_cselect_b32 s0, s5, s7
; GFX8-NEXT: s_bfe_u32 s1, s0, 0x10010
; GFX8-NEXT: s_add_i32 s1, s1, s0
-; GFX8-NEXT: s_add_i32 s5, s1, 0x7fff
-; GFX8-NEXT: s_or_b32 s7, s0, 0x400000
-; GFX8-NEXT: s_and_b64 s[0:1], s[2:3], exec
-; GFX8-NEXT: s_cselect_b32 s0, s7, s5
+; GFX8-NEXT: s_addk_i32 s1, 0x7fff
+; GFX8-NEXT: s_bitset1_b32 s0, 22
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8-NEXT: s_cselect_b32 s0, s0, s1
; GFX8-NEXT: s_lshr_b32 s7, s0, 16
; GFX8-NEXT: s_lshr_b64 s[0:1], s[6:7], 16
; GFX8-NEXT: s_mov_b32 s1, 0x7fff7fff
@@ -4523,7 +4523,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX9-NEXT: v_cvt_f64_f32_e32 v[0:1], v3
; GFX9-NEXT: v_cmp_nlg_f64_e64 s[2:3], s[0:1], v[0:1]
; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
-; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX9-NEXT: s_cselect_b32 s6, 1, -1
; GFX9-NEXT: s_add_i32 s12, s5, s6
; GFX9-NEXT: s_and_b64 s[6:7], s[10:11], exec
@@ -4533,14 +4533,14 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX9-NEXT: s_add_i32 s5, s6, s5
; GFX9-NEXT: v_cmp_gt_f64_e64 s[6:7], |s[0:1]|, |v[0:1]|
; GFX9-NEXT: s_addk_i32 s5, 0x7fff
-; GFX9-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[8:9], 0
; GFX9-NEXT: s_cselect_b32 s5, s10, s5
; GFX9-NEXT: s_lshr_b32 s5, s5, 16
; GFX9-NEXT: v_readfirstlane_b32 s10, v3
; GFX9-NEXT: s_bitcmp1_b32 s10, 0
; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-NEXT: s_or_b64 s[2:3], s[2:3], s[8:9]
-; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX9-NEXT: v_cmp_u_f64_e64 s[0:1], s[0:1], s[0:1]
; GFX9-NEXT: s_cselect_b32 s6, 1, -1
; GFX9-NEXT: s_add_i32 s6, s10, s6
@@ -4550,7 +4550,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX9-NEXT: s_add_i32 s3, s3, s2
; GFX9-NEXT: s_addk_i32 s3, 0x7fff
; GFX9-NEXT: s_bitset1_b32 s2, 22
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9-NEXT: s_cselect_b32 s0, s2, s3
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s5
@@ -4574,36 +4574,36 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX10-NEXT: v_cmp_gt_f64_e64 s5, |s[2:3]|, |v[0:1]|
; GFX10-NEXT: v_cmp_u_f64_e64 s3, s[2:3], s[2:3]
; GFX10-NEXT: v_cmp_nlg_f64_e64 s2, s[0:1], v[2:3]
-; GFX10-NEXT: v_cmp_gt_f64_e64 s8, |s[0:1]|, |v[2:3]|
-; GFX10-NEXT: v_cmp_u_f64_e64 s0, s[0:1], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_or_b32 s7, vcc_lo, s7
-; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX10-NEXT: s_cselect_b32 s5, 1, -1
-; GFX10-NEXT: s_add_i32 s5, s6, s5
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
+; GFX10-NEXT: v_cmp_gt_f64_e64 s5, |s[0:1]|, |v[2:3]|
+; GFX10-NEXT: s_cselect_b32 s8, 1, -1
+; GFX10-NEXT: v_cmp_u_f64_e64 s0, s[0:1], s[0:1]
+; GFX10-NEXT: s_add_i32 s8, s6, s8
; GFX10-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX10-NEXT: s_cselect_b32 s5, s6, s5
-; GFX10-NEXT: v_readfirstlane_b32 s6, v5
-; GFX10-NEXT: s_bfe_u32 s1, s5, 0x10010
-; GFX10-NEXT: s_add_i32 s1, s1, s5
-; GFX10-NEXT: s_bitset1_b32 s5, 22
-; GFX10-NEXT: s_addk_i32 s1, 0x7fff
-; GFX10-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX10-NEXT: s_cselect_b32 s1, s5, s1
+; GFX10-NEXT: s_cselect_b32 s6, s6, s8
+; GFX10-NEXT: v_readfirstlane_b32 s8, v5
+; GFX10-NEXT: s_bfe_u32 s7, s6, 0x10010
+; GFX10-NEXT: s_add_i32 s7, s7, s6
+; GFX10-NEXT: s_bitset1_b32 s6, 22
+; GFX10-NEXT: s_addk_i32 s7, 0x7fff
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10-NEXT: s_cselect_b32 s1, s6, s7
; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: s_bitcmp1_b32 s6, 0
+; GFX10-NEXT: s_bitcmp1_b32 s8, 0
; GFX10-NEXT: s_cselect_b32 s3, -1, 0
; GFX10-NEXT: s_or_b32 s2, s2, s3
-; GFX10-NEXT: s_and_b32 s3, s8, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s5, 0
; GFX10-NEXT: s_cselect_b32 s3, 1, -1
-; GFX10-NEXT: s_add_i32 s3, s6, s3
+; GFX10-NEXT: s_add_i32 s3, s8, s3
; GFX10-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX10-NEXT: s_cselect_b32 s2, s6, s3
+; GFX10-NEXT: s_cselect_b32 s2, s8, s3
; GFX10-NEXT: s_bfe_u32 s3, s2, 0x10010
; GFX10-NEXT: s_add_i32 s3, s3, s2
; GFX10-NEXT: s_bitset1_b32 s2, 22
; GFX10-NEXT: s_addk_i32 s3, 0x7fff
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s0, 0
; GFX10-NEXT: s_cselect_b32 s0, s2, s3
; GFX10-NEXT: s_lshr_b32 s0, s0, 16
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
@@ -4620,16 +4620,16 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX11-NEXT: v_cvt_f64_f32_e32 v[2:3], v5
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
; GFX11-NEXT: s_bitcmp1_b32 s7, 0
+; GFX11-NEXT: s_cselect_b32 s8, -1, 0
; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[2:3], v[0:1]
; GFX11-NEXT: v_cmp_gt_f64_e64 s5, |s[2:3]|, |v[0:1]|
; GFX11-NEXT: v_cmp_u_f64_e64 s3, s[2:3], s[2:3]
; GFX11-NEXT: v_cmp_nlg_f64_e64 s2, s[0:1], v[2:3]
; GFX11-NEXT: v_cmp_gt_f64_e64 s6, |s[0:1]|, |v[2:3]|
; GFX11-NEXT: v_cmp_u_f64_e64 s0, s[0:1], s[0:1]
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
; GFX11-NEXT: v_mov_b32_e32 v0, s4
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
-; GFX11-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX11-NEXT: s_or_b32 s1, vcc_lo, s8
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
; GFX11-NEXT: s_cselect_b32 s5, 1, -1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_add_i32 s5, s7, s5
@@ -4640,14 +4640,14 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX11-NEXT: s_add_i32 s5, s5, s1
; GFX11-NEXT: s_bitset1_b32 s1, 22
; GFX11-NEXT: s_addk_i32 s5, 0x7fff
-; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_cselect_b32 s1, s1, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s1, s1, 16
; GFX11-NEXT: s_bitcmp1_b32 s7, 0
; GFX11-NEXT: s_cselect_b32 s3, -1, 0
; GFX11-NEXT: s_or_b32 s2, s2, s3
-; GFX11-NEXT: s_and_b32 s3, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_cselect_b32 s3, 1, -1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_add_i32 s3, s7, s3
@@ -4658,7 +4658,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> i
; GFX11-NEXT: s_add_i32 s3, s3, s2
; GFX11-NEXT: s_bitset1_b32 s2, 22
; GFX11-NEXT: s_addk_i32 s3, 0x7fff
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_cselect_b32 s0, s2, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -4714,13 +4714,13 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2bf16_sign_v2f32(<2 x bfloat> i
; GFX8-NEXT: s_add_i32 s3, s3, s1
; GFX8-NEXT: s_addk_i32 s3, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], s1, s1
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8-NEXT: s_cselect_b32 s4, s1, s3
; GFX8-NEXT: s_bfe_u32 s1, s2, 0x10010
; GFX8-NEXT: s_add_i32 s1, s1, s2
; GFX8-NEXT: s_addk_i32 s1, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[6:7], s2, s2
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX8-NEXT: s_cselect_b32 s1, s2, s1
; GFX8-NEXT: s_lshr_b32 s5, s1, 16
; GFX8-NEXT: s_lshr_b64 s[2:3], s[4:5], 16
@@ -4738,7 +4738,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2bf16_sign_v2f32(<2 x bfloat> i
; GFX9-NEXT: s_or_b32 s4, s2, 0x400000
; GFX9-NEXT: s_add_i32 s5, s3, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[2:3], s2, s2
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cselect_b32 s2, s4, s5
; GFX9-NEXT: s_lshr_b32 s4, s2, 16
; GFX9-NEXT: s_bfe_u32 s2, s1, 0x10010
@@ -4746,7 +4746,7 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2bf16_sign_v2f32(<2 x bfloat> i
; GFX9-NEXT: s_or_b32 s5, s1, 0x400000
; GFX9-NEXT: s_add_i32 s6, s2, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s1
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cselect_b32 s1, s5, s6
; GFX9-NEXT: s_lshr_b32 s1, s1, 16
; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
@@ -4764,15 +4764,15 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2bf16_sign_v2f32(<2 x bfloat> i
; GFX10-NEXT: s_add_i32 s3, s3, s2
; GFX10-NEXT: s_bitset1_b32 s2, 22
; GFX10-NEXT: s_addk_i32 s3, 0x7fff
-; GFX10-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, s1, s1
; GFX10-NEXT: s_cselect_b32 s2, s2, s3
; GFX10-NEXT: s_bfe_u32 s3, s1, 0x10010
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, s1, s1
-; GFX10-NEXT: s_add_i32 s3, s3, s1
; GFX10-NEXT: s_lshr_b32 s2, s2, 16
+; GFX10-NEXT: s_add_i32 s3, s3, s1
; GFX10-NEXT: s_bitset1_b32 s1, 22
; GFX10-NEXT: s_addk_i32 s3, 0x7fff
-; GFX10-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: s_cselect_b32 s1, s1, s3
; GFX10-NEXT: s_lshr_b32 s1, s1, 16
; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s2
@@ -4788,15 +4788,15 @@ define amdgpu_ps i32 @s_copysign_out_v2bf16_mag_v2bf16_sign_v2f32(<2 x bfloat> i
; GFX11-NEXT: s_add_i32 s3, s3, s2
; GFX11-NEXT: s_bitset1_b32 s2, 22
; GFX11-NEXT: s_addk_i32 s3, 0x7fff
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: v_cmp_u_f32_e64 s4, s1, s1
; GFX11-NEXT: s_cselect_b32 s2, s2, s3
; GFX11-NEXT: s_bfe_u32 s3, s1, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s4, s1, s1
-; GFX11-NEXT: s_add_i32 s3, s3, s1
; GFX11-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-NEXT: s_add_i32 s3, s3, s1
; GFX11-NEXT: s_bitset1_b32 s1, 22
; GFX11-NEXT: s_addk_i32 s3, 0x7fff
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_cselect_b32 s1, s1, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s1, s1, 16
@@ -7408,7 +7408,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f32(float inreg %sign) {
; GFX8-NEXT: s_add_i32 s1, s1, s0
; GFX8-NEXT: s_addk_i32 s1, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[2:3], s0, s0
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cselect_b32 s0, s0, s1
; GFX8-NEXT: s_lshr_b32 s0, s0, 16
; GFX8-NEXT: s_and_b32 s0, s0, 0x8000
@@ -7420,7 +7420,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f32(float inreg %sign) {
; GFX9-NEXT: s_add_i32 s1, s1, s0
; GFX9-NEXT: s_addk_i32 s1, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[2:3], s0, s0
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cselect_b32 s0, s0, s1
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: s_and_b32 s0, s0, 0x8000
@@ -7432,7 +7432,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f32(float inreg %sign) {
; GFX10-NEXT: v_cmp_u_f32_e64 s2, s0, s0
; GFX10-NEXT: s_add_i32 s1, s1, s0
; GFX10-NEXT: s_addk_i32 s1, 0x7fff
-; GFX10-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: s_cselect_b32 s0, s0, s1
; GFX10-NEXT: s_lshr_b32 s0, s0, 16
; GFX10-NEXT: s_and_b32 s0, s0, 0x8000
@@ -7445,7 +7445,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f32(float inreg %sign) {
; GFX11-NEXT: s_add_i32 s1, s1, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_addk_i32 s1, 0x7fff
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s0, s0, s1
; GFX11-NEXT: s_lshr_b32 s0, s0, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7542,7 +7542,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX8-NEXT: v_cmp_gt_f64_e64 s[2:3], |s[0:1]|, |v[0:1]|
; GFX8-NEXT: v_cmp_u_f64_e64 s[0:1], s[0:1], s[0:1]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cselect_b32 s2, 1, -1
; GFX8-NEXT: s_add_i32 s7, s6, s2
; GFX8-NEXT: s_and_b64 s[2:3], s[4:5], exec
@@ -7550,7 +7550,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX8-NEXT: s_bfe_u32 s3, s2, 0x10010
; GFX8-NEXT: s_add_i32 s3, s3, s2
; GFX8-NEXT: s_addk_i32 s3, 0x7fff
-; GFX8-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX8-NEXT: s_cselect_b32 s0, s2, s3
; GFX8-NEXT: s_lshr_b32 s0, s0, 16
; GFX8-NEXT: s_and_b32 s0, s0, 0x8000
@@ -7567,7 +7567,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX9-NEXT: v_cmp_gt_f64_e64 s[2:3], |s[0:1]|, |v[0:1]|
; GFX9-NEXT: v_cmp_u_f64_e64 s[0:1], s[0:1], s[0:1]
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cselect_b32 s2, 1, -1
; GFX9-NEXT: s_add_i32 s7, s6, s2
; GFX9-NEXT: s_and_b64 s[2:3], s[4:5], exec
@@ -7575,7 +7575,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX9-NEXT: s_bfe_u32 s3, s2, 0x10010
; GFX9-NEXT: s_add_i32 s3, s3, s2
; GFX9-NEXT: s_addk_i32 s3, 0x7fff
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9-NEXT: s_cselect_b32 s0, s2, s3
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: s_and_b32 s0, s0, 0x8000
@@ -7592,7 +7592,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX10-NEXT: v_cmp_u_f64_e64 s0, s[0:1], s[0:1]
; GFX10-NEXT: s_cselect_b32 s1, -1, 0
; GFX10-NEXT: s_or_b32 s1, vcc_lo, s1
-; GFX10-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: s_cselect_b32 s2, 1, -1
; GFX10-NEXT: s_add_i32 s2, s3, s2
; GFX10-NEXT: s_and_b32 s1, s1, exec_lo
@@ -7600,7 +7600,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX10-NEXT: s_bfe_u32 s2, s1, 0x10010
; GFX10-NEXT: s_add_i32 s2, s2, s1
; GFX10-NEXT: s_addk_i32 s2, 0x7fff
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s0, 0
; GFX10-NEXT: s_cselect_b32 s0, s1, s2
; GFX10-NEXT: s_lshr_b32 s0, s0, 16
; GFX10-NEXT: s_and_b32 s0, s0, 0x8000
@@ -7619,7 +7619,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX11-NEXT: s_cselect_b32 s3, -1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, -1
; GFX11-NEXT: s_add_i32 s2, s1, s2
; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
@@ -7629,7 +7629,7 @@ define amdgpu_ps i32 @s_copysign_bf16_0_f64(double inreg %sign) {
; GFX11-NEXT: s_add_i32 s2, s2, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_addk_i32 s2, 0x7fff
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_cselect_b32 s0, s1, s2
; GFX11-NEXT: s_lshr_b32 s0, s0, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -7804,13 +7804,13 @@ define amdgpu_ps i32 @s_copysign_v2bf16_0_v2f32(<2 x float> inreg %sign) {
; GFX8-NEXT: s_add_i32 s2, s2, s0
; GFX8-NEXT: s_add_i32 s4, s2, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[2:3], s0, s0
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cselect_b32 s0, s0, s4
; GFX8-NEXT: s_bfe_u32 s2, s1, 0x10010
; GFX8-NEXT: s_add_i32 s2, s2, s1
; GFX8-NEXT: s_add_i32 s4, s2, 0x7fff
; GFX8-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s1
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8-NEXT: s_cselect_b32 s1, s1, s4
; GFX8-NEXT: s_lshr_b32 s1, s1, 16
; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], 16
@@ -7827,7 +7827,7 @@ define amdgpu_ps i32 @s_copysign_v2bf16_0_v2f32(<2 x float> inreg %sign) {
; GFX9-NEXT: s_or_b32 s4, s1, 0x400000
; GFX9-NEXT: s_add_i32 s5, s2, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s1
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9-NEXT: s_cselect_b32 s1, s4, s5
; GFX9-NEXT: s_lshr_b32 s2, s1, 16
; GFX9-NEXT: s_bfe_u32 s1, s0, 0x10010
@@ -7835,7 +7835,7 @@ define amdgpu_ps i32 @s_copysign_v2bf16_0_v2f32(<2 x float> inreg %sign) {
; GFX9-NEXT: s_or_b32 s3, s0, 0x400000
; GFX9-NEXT: s_add_i32 s4, s1, 0x7fff
; GFX9-NEXT: v_cmp_u_f32_e64 s[0:1], s0, s0
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9-NEXT: s_cselect_b32 s0, s3, s4
; GFX9-NEXT: s_lshr_b32 s0, s0, 16
; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
@@ -7849,15 +7849,15 @@ define amdgpu_ps i32 @s_copysign_v2bf16_0_v2f32(<2 x float> inreg %sign) {
; GFX10-NEXT: s_add_i32 s2, s2, s1
; GFX10-NEXT: s_bitset1_b32 s1, 22
; GFX10-NEXT: s_addk_i32 s2, 0x7fff
-; GFX10-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10-NEXT: v_cmp_u_f32_e64 s3, s0, s0
; GFX10-NEXT: s_cselect_b32 s1, s1, s2
; GFX10-NEXT: s_bfe_u32 s2, s0, 0x10010
-; GFX10-NEXT: v_cmp_u_f32_e64 s3, s0, s0
-; GFX10-NEXT: s_add_i32 s2, s2, s0
; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_add_i32 s2, s2, s0
; GFX10-NEXT: s_bitset1_b32 s0, 22
; GFX10-NEXT: s_addk_i32 s2, 0x7fff
-; GFX10-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s3, 0
; GFX10-NEXT: s_cselect_b32 s0, s0, s2
; GFX10-NEXT: s_lshr_b32 s0, s0, 16
; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
@@ -7871,15 +7871,15 @@ define amdgpu_ps i32 @s_copysign_v2bf16_0_v2f32(<2 x float> inreg %sign) {
; GFX11-NEXT: s_add_i32 s2, s2, s1
; GFX11-NEXT: s_bitset1_b32 s1, 22
; GFX11-NEXT: s_addk_i32 s2, 0x7fff
-; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-NEXT: v_cmp_u_f32_e64 s3, s0, s0
; GFX11-NEXT: s_cselect_b32 s1, s1, s2
; GFX11-NEXT: s_bfe_u32 s2, s0, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s3, s0, s0
-; GFX11-NEXT: s_add_i32 s2, s2, s0
; GFX11-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-NEXT: s_add_i32 s2, s2, s0
; GFX11-NEXT: s_bitset1_b32 s0, 22
; GFX11-NEXT: s_addk_i32 s2, 0x7fff
-; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-NEXT: s_cselect_b32 s0, s0, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s0, s0, 16
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
new file mode 100644
index 00000000000000..db16640cadbd11
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
@@ -0,0 +1,59 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-fix-sgpr-copies -o - %s | FileCheck -check-prefixes=CHECK,GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=bonaire -run-pass=si-fix-sgpr-copies -o - %s | FileCheck -check-prefixes=CHECK,GFX7 %s
+
+# A copy of a lane mask to SCC is lowered to an AND with EXEC, whose only
+# purpose is to define SCC. When the lane mask already has 0 in the bits of
+# all inactive lanes, S_CMP computes the same SCC without needing a
+# destination register.
+
+---
+name: vcmp_to_scc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: vcmp_to_scc
+ ; CHECK: [[CMP:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ ; GFX9-NEXT: S_CMP_LG_U64 [[CMP]], 0, implicit-def $scc
+ ; GFX7-NEXT: {{%[0-9]+}}:sreg_64 = S_AND_B64 [[CMP]], $exec, implicit-def $scc
+ %0:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ $scc = COPY %0
+ %1:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
+ S_ENDPGM 0, implicit %1
+...
+
+# Negative test: nothing is known about the lane mask, so it must be masked
+# with EXEC.
+---
+name: unknown_mask_to_scc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0_sgpr1
+ ; CHECK-LABEL: name: unknown_mask_to_scc
+ ; CHECK: [[MASK:%[0-9]+]]:sreg_64_xexec = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: {{%[0-9]+}}:sreg_64 = S_AND_B64 [[MASK]], $exec, implicit-def $scc
+ %0:sreg_64_xexec = COPY $sgpr0_sgpr1
+ $scc = COPY %0
+ %1:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
+ S_ENDPGM 0, implicit %1
+...
+
+# Negative test: EXEC is written between the V_CMP and the copy to SCC, so the
+# V_CMP result may have bits set for lanes that are now inactive.
+---
+name: exec_written_in_between
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+ ; CHECK-LABEL: name: exec_written_in_between
+ ; CHECK: [[CMP:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, $sgpr0_sgpr1, implicit-def dead $scc
+ ; CHECK-NEXT: {{%[0-9]+}}:sreg_64 = S_AND_B64 [[CMP]], $exec, implicit-def $scc
+ %0:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ $exec = S_OR_B64 $exec, $sgpr0_sgpr1, implicit-def dead $scc
+ $scc = COPY %0
+ %1:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
+ S_ENDPGM 0, implicit %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
index 6fd6e0d072e36f..20d76ce863652a 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
@@ -624,12 +624,12 @@ define amdgpu_ps double @fneg_fadd_0_f64(double inreg %tmp2, double inreg %tmp6,
; VI-NEXT: v_add_f64 v[0:1], v[0:1], 0
; VI-NEXT: v_cmp_ngt_f64_e32 vcc, s[0:1], v[0:1]
; VI-NEXT: v_xor_b32_e32 v3, 0x80000000, v1
+; VI-NEXT: s_mov_b32 s0, 0
; VI-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; VI-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s1, 0, 0x7ff80000
-; VI-NEXT: s_mov_b32 s0, 0
; VI-NEXT: ; return to shader part epilog
.entry:
%tmp7 = fdiv double 1.000000e+00, %tmp6
@@ -691,12 +691,12 @@ define amdgpu_ps double @fneg_fadd_0_f64_nsz(double inreg %tmp2, double inreg %t
; VI-NEXT: s_brev_b32 s3, 1
; VI-NEXT: v_mul_f64 v[0:1], v[0:1], s[2:3]
; VI-NEXT: v_cmp_nlt_f64_e64 vcc, -v[0:1], s[0:1]
+; VI-NEXT: s_mov_b32 s0, 0
; VI-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; VI-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s1, 0, 0x7ff80000
-; VI-NEXT: s_mov_b32 s0, 0
; VI-NEXT: ; return to shader part epilog
.entry:
%tmp7 = fdiv double 1.000000e+00, %tmp6
@@ -749,12 +749,12 @@ define amdgpu_ps double @fneg_fadd_0_nsz_f64(double inreg %tmp2, double inreg %t
; VI-NEXT: v_mov_b32_e32 v3, s0
; VI-NEXT: v_mul_f64 v[0:1], v[0:1], s[2:3]
; VI-NEXT: v_cmp_nlt_f64_e64 vcc, -v[0:1], s[0:1]
+; VI-NEXT: s_mov_b32 s0, 0
; VI-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; VI-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s1, 0, 0x7ff80000
-; VI-NEXT: s_mov_b32 s0, 0
; VI-NEXT: ; return to shader part epilog
.entry:
%tmp7 = fdiv afn double 1.000000e+00, %tmp6
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 1a3bd8b68f9644..f354f545120027 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -1686,32 +1686,49 @@ define amdgpu_kernel void @multiple_uses_fneg_select_f64(double %x, double %y, i
}
define amdgpu_kernel void @fnge_select_f32_multi_use_regression(float %.i2369) {
-; GCN-LABEL: fnge_select_f32_multi_use_regression:
-; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_load_dword s0, s[8:9], 0x0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_nlt_f32_e64 s[0:1], s0, 0
-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT: s_cselect_b32 s2, 1, 0
-; GCN-NEXT: v_cmp_gt_f32_e64 s[0:1], s2, 0
-; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GCN-NEXT: s_cselect_b32 s0, s2, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mul_f32_e64 v0, -s2, v0
-; GCN-NEXT: v_cmp_le_f32_e32 vcc, 0, v0
-; GCN-NEXT: s_and_b64 vcc, exec, vcc
-; GCN-NEXT: s_endpgm
+;
+; GFX7-LABEL: fnge_select_f32_multi_use_regression:
+; GFX7: ; %bb.0: ; %.entry
+; GFX7-NEXT: s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_cmp_nlt_f32_e64 s[0:1], s0, 0
+; GFX7-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX7-NEXT: s_cselect_b32 s2, 1, 0
+; GFX7-NEXT: v_cmp_gt_f32_e64 s[0:1], s2, 0
+; GFX7-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX7-NEXT: s_cselect_b32 s0, s2, 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX7-NEXT: v_cmp_le_f32_e32 vcc, 0, v0
+; GFX7-NEXT: s_and_b64 vcc, exec, vcc
+; GFX7-NEXT: s_endpgm
+;
+; GFX9-LABEL: fnge_select_f32_multi_use_regression:
+; GFX9: ; %bb.0: ; %.entry
+; GFX9-NEXT: s_load_dword s0, s[8:9], 0x0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_cmp_nlt_f32_e64 s[0:1], s0, 0
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: v_cmp_gt_f32_e64 s[0:1], s2, 0
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: s_cselect_b32 s0, s2, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX9-NEXT: v_cmp_le_f32_e32 vcc, 0, v0
+; GFX9-NEXT: s_and_b64 vcc, exec, vcc
+; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: fnge_select_f32_multi_use_regression:
; GFX11: ; %bb.0: ; %.entry
; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_nlt_f32_e64 s0, s0, 0
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_cselect_b32 s0, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_cmp_gt_f32_e64 s1, s0, 0
-; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_cselect_b32 s1, s0, 0
; GFX11-NEXT: v_mul_f32_e64 v0, -s0, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/fp-classify.ll b/llvm/test/CodeGen/AMDGPU/fp-classify.ll
index 6811f71f14b25d..a357142925b879 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-classify.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-classify.ll
@@ -30,7 +30,7 @@ define amdgpu_kernel void @test_isinf_pattern(ptr addrspace(1) nocapture %out, f
; VI-NEXT: v_mov_b32_e32 v0, 0x204
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -45,7 +45,7 @@ define amdgpu_kernel void @test_isinf_pattern(ptr addrspace(1) nocapture %out, f
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x204
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -81,7 +81,7 @@ define amdgpu_kernel void @test_not_isinf_pattern_0(ptr addrspace(1) nocapture %
; VI-NEXT: v_mov_b32_e32 v0, 0x7f800000
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_nlg_f32_e64 s[2:3], |s2|, v0
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -96,7 +96,7 @@ define amdgpu_kernel void @test_not_isinf_pattern_0(ptr addrspace(1) nocapture %
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_nlg_f32_e64 s2, 0x7f800000, |s2|
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -167,7 +167,7 @@ define amdgpu_kernel void @test_isfinite_pattern_0(ptr addrspace(1) nocapture %o
; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -182,7 +182,7 @@ define amdgpu_kernel void @test_isfinite_pattern_0(ptr addrspace(1) nocapture %o
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -220,7 +220,7 @@ define amdgpu_kernel void @test_isfinite_pattern_1(ptr addrspace(1) nocapture %o
; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -235,7 +235,7 @@ define amdgpu_kernel void @test_isfinite_pattern_1(ptr addrspace(1) nocapture %o
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -270,7 +270,7 @@ define amdgpu_kernel void @test_isfinite_not_pattern_0(ptr addrspace(1) nocaptur
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_o_f32_e64 s[2:3], s2, s2
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -285,7 +285,7 @@ define amdgpu_kernel void @test_isfinite_not_pattern_0(ptr addrspace(1) nocaptur
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_o_f32_e64 s2, s2, s2
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -328,7 +328,6 @@ define amdgpu_kernel void @test_isfinite_not_pattern_1(ptr addrspace(1) nocaptur
; VI-NEXT: v_cmp_o_f32_e64 s[2:3], s6, s6
; VI-NEXT: v_cmp_neq_f32_e32 vcc, s6, v0
; VI-NEXT: s_and_b64 s[2:3], s[2:3], vcc
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -345,9 +344,8 @@ define amdgpu_kernel void @test_isfinite_not_pattern_1(ptr addrspace(1) nocaptur
; GFX11-NEXT: v_cmp_o_f32_e64 s3, s2, s2
; GFX11-NEXT: v_cmp_neq_f32_e64 s2, 0x7f800000, s2
; GFX11-NEXT: s_and_b32 s2, s3, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
@@ -387,7 +385,6 @@ define amdgpu_kernel void @test_isfinite_not_pattern_2(ptr addrspace(1) nocaptur
; VI-NEXT: v_cmp_o_f32_e64 s[4:5], s2, s2
; VI-NEXT: v_cmp_neq_f32_e64 s[2:3], |s3|, v0
; VI-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -402,9 +399,8 @@ define amdgpu_kernel void @test_isfinite_not_pattern_2(ptr addrspace(1) nocaptur
; GFX11-NEXT: v_cmp_o_f32_e64 s2, s2, s2
; GFX11-NEXT: v_cmp_neq_f32_e64 s3, 0x7f800000, |s3|
; GFX11-NEXT: s_and_b32 s2, s2, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
@@ -445,7 +441,6 @@ define amdgpu_kernel void @test_isfinite_not_pattern_3(ptr addrspace(1) nocaptur
; VI-NEXT: v_cmp_u_f32_e64 s[2:3], s6, s6
; VI-NEXT: v_cmp_neq_f32_e64 s[4:5], |s6|, v0
; VI-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -462,9 +457,8 @@ define amdgpu_kernel void @test_isfinite_not_pattern_3(ptr addrspace(1) nocaptur
; GFX11-NEXT: v_cmp_u_f32_e64 s3, s2, s2
; GFX11-NEXT: v_cmp_neq_f32_e64 s2, 0x7f800000, |s2|
; GFX11-NEXT: s_and_b32 s2, s3, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
@@ -500,7 +494,7 @@ define amdgpu_kernel void @test_isfinite_pattern_4(ptr addrspace(1) nocapture %o
; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -515,7 +509,7 @@ define amdgpu_kernel void @test_isfinite_pattern_4(ptr addrspace(1) nocapture %o
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -553,7 +547,7 @@ define amdgpu_kernel void @test_isfinite_pattern_4_commute_and(ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -568,7 +562,7 @@ define amdgpu_kernel void @test_isfinite_pattern_4_commute_and(ptr addrspace(1)
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -614,7 +608,6 @@ define amdgpu_kernel void @test_not_isfinite_pattern_4_wrong_ord_test(ptr addrsp
; VI-NEXT: v_cmp_class_f32_e32 vcc, s1, v0
; VI-NEXT: v_cmp_o_f32_e64 s[0:1], s1, v1
; VI-NEXT: s_and_b64 s[0:1], s[0:1], vcc
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
@@ -632,9 +625,8 @@ define amdgpu_kernel void @test_not_isfinite_pattern_4_wrong_ord_test(ptr addrsp
; GFX11-NEXT: v_cmp_o_f32_e64 s3, s2, s3
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
; GFX11-NEXT: s_and_b32 s2, s3, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
@@ -671,7 +663,7 @@ define amdgpu_kernel void @test_isinf_pattern_f16(ptr addrspace(1) nocapture %ou
; VI-NEXT: v_mov_b32_e32 v0, 0x204
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -686,7 +678,7 @@ define amdgpu_kernel void @test_isinf_pattern_f16(ptr addrspace(1) nocapture %ou
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f16_e64 s2, s2, 0x204
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -726,7 +718,7 @@ define amdgpu_kernel void @test_isfinite_pattern_0_f16(ptr addrspace(1) nocaptur
; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -741,7 +733,7 @@ define amdgpu_kernel void @test_isfinite_pattern_0_f16(ptr addrspace(1) nocaptur
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f16_e64 s2, s2, 0x1f8
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -780,7 +772,7 @@ define amdgpu_kernel void @test_isfinite_pattern_4_f16(ptr addrspace(1) nocaptur
; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -795,7 +787,7 @@ define amdgpu_kernel void @test_isfinite_pattern_4_f16(ptr addrspace(1) nocaptur
; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f16_e64 s2, s2, 0x1f8
-; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
index 2618c670b8c23a..d2eaf783f95840 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
@@ -1110,7 +1110,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], -1.0, s6
-; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: s_cselect_b32 s4, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -1123,7 +1123,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, -1.0, s2
-; GFX11-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-SDAG-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -1193,7 +1193,7 @@ define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, floa
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], -1.0, |s6|
-; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: s_cselect_b32 s4, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -1206,7 +1206,7 @@ define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, floa
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, -1.0, |s2|
-; GFX11-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-SDAG-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
index 2fb9e0b764391d..dce65ceb964dda 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
@@ -881,7 +881,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], 1.0, s6
-; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: s_cselect_b32 s4, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -894,7 +894,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, 1.0, s2
-; GFX11-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-SDAG-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -964,7 +964,7 @@ define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, floa
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], 1.0, |s6|
-; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: s_cselect_b32 s4, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -977,7 +977,7 @@ define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, floa
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, 1.0, |s2|
-; GFX11-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-SDAG-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
index 66b2ba892182a2..c25be90e04d381 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
@@ -709,26 +709,26 @@ define i64 @test_s_signed_i64_f32(float inreg %f) nounwind {
; GFX9-NEXT: v_readfirstlane_b32 s5, v1
; GFX9-NEXT: s_mov_b32 s7, s6
; GFX9-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX9-NEXT: s_sub_u32 s7, s4, s6
+; GFX9-NEXT: s_sub_u32 s4, s4, s6
; GFX9-NEXT: v_mov_b32_e32 v0, 0xdf000000
-; GFX9-NEXT: s_subb_u32 s10, s5, s6
+; GFX9-NEXT: s_subb_u32 s8, s5, s6
; GFX9-NEXT: v_cmp_nge_f32_e32 vcc, s16, v0
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: v_mov_b32_e32 v0, 0x5effffff
+; GFX9-NEXT: s_cselect_b32 s6, 0, s4
; GFX9-NEXT: v_cmp_gt_f32_e64 s[4:5], s16, v0
-; GFX9-NEXT: s_cselect_b32 s8, 0, s7
-; GFX9-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
+; GFX9-NEXT: s_cselect_b32 s9, -1, s6
; GFX9-NEXT: v_cmp_u_f32_e64 s[6:7], s16, s16
-; GFX9-NEXT: s_cselect_b32 s11, -1, s8
-; GFX9-NEXT: s_and_b64 s[8:9], s[6:7], exec
-; GFX9-NEXT: s_cselect_b32 s11, 0, s11
-; GFX9-NEXT: s_and_b64 s[8:9], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s8, 0x80000000, s10
-; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s8, 0x7fffffff, s8
-; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], exec
-; GFX9-NEXT: s_cselect_b32 s4, 0, s8
-; GFX9-NEXT: v_mov_b32_e32 v0, s11
+; GFX9-NEXT: s_cmp_lg_u64 s[6:7], 0
+; GFX9-NEXT: s_cselect_b32 s9, 0, s9
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s8, 0x80000000, s8
+; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
+; GFX9-NEXT: s_cselect_b32 s4, 0x7fffffff, s8
+; GFX9-NEXT: s_cmp_lg_u64 s[6:7], 0
+; GFX9-NEXT: s_cselect_b32 s4, 0, s4
+; GFX9-NEXT: v_mov_b32_e32 v0, s9
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -736,33 +736,33 @@ define i64 @test_s_signed_i64_f32(float inreg %f) nounwind {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_trunc_f32_e32 v0, s0
-; GFX11-NEXT: v_cmp_lt_f32_e64 s6, 0x5effffff, s0
; GFX11-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
; GFX11-NEXT: v_readfirstlane_b32 s1, v0
; GFX11-NEXT: v_floor_f32_e32 v1, v1
; GFX11-NEXT: s_ashr_i32 s4, s1, 31
; GFX11-NEXT: v_cmp_nle_f32_e64 s1, 0xdf000000, s0
; GFX11-NEXT: s_mov_b32 s5, s4
-; GFX11-NEXT: v_cmp_u_f32_e64 s0, s0, s0
; GFX11-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v1
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: v_cmp_lt_f32_e64 s5, 0x5effffff, s0
; GFX11-NEXT: s_sub_u32 s2, s2, s4
; GFX11-NEXT: s_subb_u32 s3, s3, s4
-; GFX11-NEXT: s_and_b32 s4, s1, exec_lo
+; GFX11-NEXT: v_cmp_u_f32_e64 s0, s0, s0
+; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_cselect_b32 s2, 0, s2
-; GFX11-NEXT: s_and_b32 s4, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
; GFX11-NEXT: s_cselect_b32 s2, -1, s2
-; GFX11-NEXT: s_and_b32 s4, s0, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_cselect_b32 s2, 0, s2
-; GFX11-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-NEXT: s_cselect_b32 s1, 0x80000000, s3
-; GFX11-NEXT: s_and_b32 s3, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s5, 0
; GFX11-NEXT: s_cselect_b32 s1, 0x7fffffff, s1
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
; GFX11-NEXT: s_cselect_b32 s0, 0, s1
; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll b/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
index 5faac8df0e0288..9860b4c64560cb 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
@@ -605,7 +605,7 @@ define amdgpu_kernel void @fptosi_f16_to_i1(ptr addrspace(1) %out, half %in) {
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_f16_e64 s[4:5], -1.0, s6
-; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: s_cselect_b32 s4, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -619,7 +619,7 @@ define amdgpu_kernel void @fptosi_f16_to_i1(ptr addrspace(1) %out, half %in) {
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e64 s2, -1.0, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -635,7 +635,7 @@ define amdgpu_kernel void @fptosi_f16_to_i1(ptr addrspace(1) %out, half %in) {
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e64 s2, -1.0, s2
-; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll b/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
index 097d89b9e0ce18..60c09cfb8dff93 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
@@ -602,7 +602,7 @@ define amdgpu_kernel void @fptoui_f16_to_i1(ptr addrspace(1) %out, half %in) {
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_f16_e64 s[4:5], 1.0, s6
-; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: s_cselect_b32 s4, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
@@ -616,7 +616,7 @@ define amdgpu_kernel void @fptoui_f16_to_i1(ptr addrspace(1) %out, half %in) {
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e64 s2, 1.0, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
@@ -632,7 +632,7 @@ define amdgpu_kernel void @fptoui_f16_to_i1(ptr addrspace(1) %out, half %in) {
; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e64 s2, 1.0, s2
-; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 112cfabd6d4343..c60c3841f3dab4 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -18165,12 +18165,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_nlg_f64_e32 vcc, 0, v[0:1]
; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: v_cmp_nlg_f64_e32 vcc, 0, v[2:3]
; VI-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, 0x7ff80000, 0
; VI-NEXT: v_mov_b32_e32 v3, s0
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
@@ -18185,13 +18185,13 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_nlg_f64_e32 vcc, 0, v[1:2]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: v_cmp_nlg_f64_e32 vcc, 0, v[3:4]
-; GFX9-NEXT: s_cselect_b32 s4, 0x7ff80000, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: s_and_b64 s[2:3], vcc, exec
; GFX9-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: global_store_dwordx4 v0, v[0:3], s[0:1]
; GFX9-NEXT: s_endpgm
;
@@ -18204,11 +18204,11 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX10-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX10-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX10-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: global_store_dwordx4 v0, v[0:3], s[0:1]
@@ -18223,12 +18223,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
; GFX11-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX11-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX11-NEXT: v_mov_b32_e32 v3, s3
; GFX11-NEXT: global_store_b128 v0, v[0:3], s[0:1]
@@ -18243,12 +18243,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX1150-NEXT: s_waitcnt vmcnt(0)
; GFX1150-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
; GFX1150-NEXT: v_mov_b32_e32 v2, v0
-; GFX1150-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX1150-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX1150-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX1150-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1150-NEXT: v_mov_b32_e32 v1, s2
-; GFX1150-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX1150-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX1150-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX1150-NEXT: v_mov_b32_e32 v3, s3
; GFX1150-NEXT: global_store_b128 v0, v[0:3], s[0:1]
@@ -18263,12 +18263,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
; GFX1200-NEXT: v_mov_b32_e32 v2, v0
-; GFX1200-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX1200-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX1200-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX1200-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1200-NEXT: v_mov_b32_e32 v1, s2
-; GFX1200-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX1200-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX1200-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: v_mov_b32_e32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
index 5cd9738a8fa902..20a3c5f4d20675 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
@@ -848,7 +848,7 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64(double inreg %x) {
; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX8-SDAG-NEXT: v_bfrev_b32_e32 v1, 8
; GFX8-SDAG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GFX8-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX8-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8-SDAG-NEXT: s_cselect_b32 s2, 0x100, 0
; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX8-SDAG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
@@ -984,7 +984,7 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64_ninf(double inreg %x) {
; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX8-SDAG-NEXT: v_bfrev_b32_e32 v1, 8
; GFX8-SDAG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GFX8-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX8-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8-SDAG-NEXT: s_cselect_b32 s2, 0x100, 0
; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX8-SDAG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
diff --git a/llvm/test/CodeGen/AMDGPU/literal64.ll b/llvm/test/CodeGen/AMDGPU/literal64.ll
index 2415fdd2c76828..b4301711522138 100644
--- a/llvm/test/CodeGen/AMDGPU/literal64.ll
+++ b/llvm/test/CodeGen/AMDGPU/literal64.ll
@@ -219,7 +219,7 @@ define i1 @class_f64() noinline optnone {
; GFX1250-SDAG-NEXT: s_mov_b32 s2, 1
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0x4063233333333333
; GFX1250-SDAG-NEXT: v_cmp_class_f64_e64 s0, s[0:1], s2
-; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX1250-SDAG-NEXT: s_cselect_b32 s0, 1, 0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -248,7 +248,7 @@ define i1 @class_f64() noinline optnone {
; GFX13-SDAG-NEXT: s_mov_b32 s2, 1
; GFX13-SDAG-NEXT: s_mov_b64 s[0:1], 0x4063233333333333
; GFX13-SDAG-NEXT: v_cmp_class_f64_e64 s0, s[0:1], s2
-; GFX13-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX13-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX13-SDAG-NEXT: s_cselect_b32 s0, 1, 0
; GFX13-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX13-SDAG-NEXT: s_set_pc_i64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
index b95212b16e5aca..90dab67613d019 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @class_f16(
; VI-SDAG-NEXT: s_mov_b32 s5, s1
; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, v0, v1
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s0, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -75,7 +75,7 @@ define amdgpu_kernel void @class_f16_fabs(
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], |s5|, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -120,7 +120,7 @@ define amdgpu_kernel void @class_f16_fneg(
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], -s5, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -165,7 +165,7 @@ define amdgpu_kernel void @class_f16_fabs_fneg(
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], -|s5|, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -209,7 +209,7 @@ define amdgpu_kernel void @class_f16_1(
; VI-SDAG-NEXT: s_mov_b32 s2, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 1
-; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -246,7 +246,7 @@ define amdgpu_kernel void @class_f16_64(
; VI-SDAG-NEXT: s_mov_b32 s2, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 64
-; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -284,7 +284,7 @@ define amdgpu_kernel void @class_f16_full_mask(
; VI-SDAG-NEXT: s_mov_b32 s2, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -323,7 +323,7 @@ define amdgpu_kernel void @class_f16_nine_bit_mask(
; VI-SDAG-NEXT: s_mov_b32 s2, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s4, -1, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 27bb8ac56706d4..307143f1c038ba 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -60,7 +60,7 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; VI-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
; VI-SDAG-NEXT: v_ldexp_f32 v2, v0, s2
@@ -98,7 +98,7 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, s2
@@ -232,17 +232,17 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-SDAG-NEXT: v_add_f32_e32 v2, s3, v2
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; VI-SDAG-NEXT: v_add_f32_e32 v2, s3, v2
; VI-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
; VI-SDAG-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, v2, s3
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
+; VI-SDAG-NEXT: v_ldexp_f32 v1, v2, s3
; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
@@ -284,7 +284,7 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, s3, v3
@@ -292,7 +292,7 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-SDAG-NEXT: v_exp_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v3, s4
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, s2
@@ -466,23 +466,23 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; VI-SDAG-NEXT: v_add_f32_e32 v2, s2, v2
-; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
; VI-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-SDAG-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT: v_ldexp_f32 v2, v2, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; VI-SDAG-NEXT: v_add_f32_e32 v3, s1, v3
; VI-SDAG-NEXT: v_add_f32_e32 v0, s0, v0
; VI-SDAG-NEXT: v_exp_f32_e32 v3, v3
; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v2, s4
; VI-SDAG-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; VI-SDAG-NEXT: v_ldexp_f32 v1, v3, s4
; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, s0
@@ -537,22 +537,22 @@ define amdgpu_kernel void @s_exp2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; GFX900-SDAG-NEXT: v_add_f32_e32 v2, s2, v2
-; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT: v_ldexp_f32 v2, v2, s2
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_add_f32_e32 v4, s1, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v0, s0, v0
; GFX900-SDAG-NEXT: v_exp_f32_e32 v4, v4
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_ldexp_f32 v2, v2, s2
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; GFX900-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v4, s2
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, s0
@@ -772,7 +772,7 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
; VI-SDAG-NEXT: v_add_f32_e32 v2, s3, v2
@@ -780,13 +780,13 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
; VI-SDAG-NEXT: v_exp_f32_e32 v4, v3
; VI-SDAG-NEXT: s_cselect_b32 s6, 0xffffffc0, 0
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; VI-SDAG-NEXT: v_ldexp_f32 v3, v2, s6
; VI-SDAG-NEXT: v_ldexp_f32 v2, v4, s2
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; VI-SDAG-NEXT: v_add_f32_e32 v4, s1, v4
@@ -794,7 +794,7 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_exp_f32_e32 v4, v4
; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
; VI-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; VI-SDAG-NEXT: v_ldexp_f32 v1, v4, s2
; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, s0
@@ -856,7 +856,7 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
; GFX900-SDAG-NEXT: v_add_f32_e32 v2, s3, v2
@@ -864,13 +864,13 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
; GFX900-SDAG-NEXT: v_exp_f32_e32 v5, v3
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, v2, s4
; GFX900-SDAG-NEXT: v_ldexp_f32 v2, v5, s2
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, s1, v5
@@ -878,7 +878,7 @@ define amdgpu_kernel void @s_exp2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-SDAG-NEXT: v_exp_f32_e32 v5, v5
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; GFX900-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, v5, s2
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index 037384edd09062..1ed1f8c539251c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -50,7 +50,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX8SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX8SELDAG-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
-; GFX8SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8SELDAG-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX8SELDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, s1
@@ -79,7 +79,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX9SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9SELDAG-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
-; GFX9SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9SELDAG-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX9SELDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX9SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
@@ -106,7 +106,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10SELDAG-NEXT: v_cmp_class_f16_e64 s2, s2, 3
-; GFX10SELDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10SELDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX10SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX10SELDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX10SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
@@ -135,7 +135,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, s2, v0.l
; GFX11SELDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11SELDAG-TRUE16-NEXT: s_and_b32 s2, vcc_lo, exec_lo
+; GFX11SELDAG-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
; GFX11SELDAG-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11SELDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s2
; GFX11SELDAG-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -148,7 +148,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX11SELDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11SELDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-FAKE16-NEXT: v_cmp_class_f16_e64 s2, s2, 3
-; GFX11SELDAG-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11SELDAG-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11SELDAG-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX11SELDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11SELDAG-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
index 17f7a401e96c84..5deda210ede1d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
@@ -48,7 +48,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
; GFX8SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX8SELDAG-NEXT: v_cmp_class_f32_e64 s[2:3], s2, 3
-; GFX8SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX8SELDAG-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX8SELDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, s1
@@ -77,7 +77,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
; GFX9SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9SELDAG-NEXT: v_cmp_class_f32_e64 s[2:3], s2, 3
-; GFX9SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9SELDAG-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX9SELDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX9SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
@@ -104,7 +104,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10SELDAG-NEXT: v_cmp_class_f32_e64 s2, s2, 3
-; GFX10SELDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10SELDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX10SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX10SELDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX10SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
@@ -131,7 +131,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
; GFX11SELDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s2, s2, 3
-; GFX11SELDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11SELDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX11SELDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11SELDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
@@ -192,7 +192,7 @@ define amdgpu_kernel void @sgpr_isnan_f64(ptr addrspace(1) %out, double %x) {
; GFX8SELDAG-NEXT: v_cmp_class_f64_e64 s[2:3], s[2:3], 3
; GFX8SELDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX8SELDAG-NEXT: s_and_b64 s[0:1], s[2:3], exec
+; GFX8SELDAG-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8SELDAG-NEXT: s_cselect_b32 s0, -1, 0
; GFX8SELDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX8SELDAG-NEXT: flat_store_dword v[0:1], v2
@@ -217,7 +217,7 @@ define amdgpu_kernel void @sgpr_isnan_f64(ptr addrspace(1) %out, double %x) {
; GFX9SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9SELDAG-NEXT: v_cmp_class_f64_e64 s[2:3], s[2:3], 3
-; GFX9SELDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9SELDAG-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX9SELDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX9SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
@@ -241,7 +241,7 @@ define amdgpu_kernel void @sgpr_isnan_f64(ptr addrspace(1) %out, double %x) {
; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10SELDAG-NEXT: v_cmp_class_f64_e64 s2, s[2:3], 3
-; GFX10SELDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10SELDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX10SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX10SELDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX10SELDAG-NEXT: global_store_dword v0, v1, s[0:1]
@@ -264,7 +264,7 @@ define amdgpu_kernel void @sgpr_isnan_f64(ptr addrspace(1) %out, double %x) {
; GFX11SELDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s2, s[2:3], 3
-; GFX11SELDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11SELDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11SELDAG-NEXT: s_cselect_b32 s2, -1, 0
; GFX11SELDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11SELDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index 0b8c2a75d68069..4b166a0b8e09ff 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -80,19 +80,19 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_log_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
+; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v2
@@ -154,18 +154,18 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_log_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3f317217
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3377d1cf
@@ -219,23 +219,23 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317217, v1
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v3, 0x3f317217, v1, -v2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fmamk_f32 v3, v1, 0x3377d1cf, v3
-; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
@@ -413,19 +413,19 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; VI-SDAG-LABEL: s_log_contract_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
+; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v2
@@ -487,18 +487,18 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; GFX900-SDAG-LABEL: s_log_contract_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3f317217
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3377d1cf
@@ -552,23 +552,23 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317217, v1
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v3, 0x3f317217, v1, -v2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fmamk_f32 v3, v1, 0x3377d1cf, v3
-; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
@@ -787,14 +787,14 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s4
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
@@ -900,38 +900,38 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; GFX900-SDAG-NEXT: s_mov_b32 s6, 0x3377d1cf
-; GFX900-SDAG-NEXT: s_mov_b32 s7, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x7f800000
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, s4
; GFX900-SDAG-NEXT: v_ldexp_f32 v4, s3, v4
; GFX900-SDAG-NEXT: v_log_f32_e32 v4, v4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3f317217
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317217, v4
-; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s6, 32, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3377d1cf
; GFX900-SDAG-NEXT: v_fma_f32 v6, v4, s3, -v5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v4, s6, v6
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v4, s4, v6
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-SDAG-NEXT: v_log_f32_e32 v6, v1
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s7
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v6
; GFX900-SDAG-NEXT: v_fma_f32 v4, v6, s3, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v6, s6, v4
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v6, s4, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s7
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v3, v0
; GFX900-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
@@ -996,33 +996,32 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s4, 0x800000, s3
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s5, 0x800000, s2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s4
-; GFX1100-SDAG-NEXT: s_and_b32 s4, s4, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s4, 32, 0
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, s5
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s5, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s3, s4
-; GFX1100-SDAG-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s5, 32, 0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s5
+; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, s5
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s3
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v3, v3
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317217, v1
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317217, v3
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_fma_f32 v6, 0x3f317217, v1, -v4
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v7, 0x3f317217, v3, -v5
-; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v6, 0x3377d1cf, v1 :: v_dual_fmac_f32 v7, 0x3377d1cf, v3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v6, 0x3377d1cf, v1 :: v_dual_fmac_f32 v7, 0x3377d1cf, v3
; GFX1100-SDAG-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_add_f32 v5, v5, v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v4 :: v_dual_mov_b32 v4, 0
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v3|
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_dual_sub_f32 v1, v1, v0 :: v_dual_sub_f32 v0, v3, v2
; GFX1100-SDAG-NEXT: global_store_b64 v4, v[0:1], s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
@@ -1336,37 +1335,37 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
; VI-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3805fdf4, v5
-; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; VI-SDAG-NEXT: s_cselect_b32 s5, 32, 0
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, s5
; VI-SDAG-NEXT: v_ldexp_f32 v6, s1, v6
; VI-SDAG-NEXT: v_log_f32_e32 v6, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
-; VI-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; VI-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
; VI-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s6
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: v_and_b32_e32 v3, 0xfffff000, v6
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_sub_f32_e32 v4, v6, v3
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3805fdf4, v3
@@ -1379,7 +1378,7 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
; VI-SDAG-NEXT: v_log_f32_e32 v4, v1
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s6
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v5
; VI-SDAG-NEXT: v_and_b32_e32 v3, 0xfffff000, v4
@@ -1391,7 +1390,7 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317000, v3
; VI-SDAG-NEXT: v_add_f32_e32 v3, v3, v5
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s6
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v0, v3, v0
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -1499,50 +1498,50 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; GFX900-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; GFX900-SDAG-NEXT: s_mov_b32 s8, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s3
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
; GFX900-SDAG-NEXT: v_log_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s5, 32, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3f317217
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317217, v3
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s2
-; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x3377d1cf
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s4, -v5
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3377d1cf
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s2, -v5
; GFX900-SDAG-NEXT: v_ldexp_f32 v7, s1, v7
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s5, v6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s3, v6
; GFX900-SDAG-NEXT: v_log_f32_e32 v7, v7
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s8
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v7
; GFX900-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s4, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s2, -v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s5, v5
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s3, v5
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v5
; GFX900-SDAG-NEXT: v_log_f32_e32 v5, v1
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s8
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v6
; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v5
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s4, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s5, v6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s2, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s3, v6
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s8
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v3, v0
@@ -1632,34 +1631,35 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s1
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s3
-; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s3, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, s6
-; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s7, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s2, s7
; GFX1100-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s2, s3
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s3
; GFX1100-SDAG-NEXT: v_ldexp_f32 v4, s1, s6
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 0x41b17218, s7
-; GFX1100-SDAG-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s0
; GFX1100-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v4, v4
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 0x41b17218, s3
+; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v5, s0, s2
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_log_f32_e32 v5, v5
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-NEXT: v_dual_mul_f32 v6, 0x3f317217, v2 :: v_dual_mul_f32 v7, 0x3f317217, v4
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v6, 0x3f317217, v2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
+; GFX1100-SDAG-NEXT: v_log_f32_e32 v5, v5
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317217, v4
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_fma_f32 v9, 0x3f317217, v2, -v6
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v10, 0x3f317217, v4, -v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v9, 0x3377d1cf, v2 :: v_dual_fmac_f32 v10, 0x3377d1cf, v4
+; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v8, 0x3f317217, v5
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v7, v7, v10
; GFX1100-SDAG-NEXT: v_fma_f32 v11, 0x3f317217, v5, -v8
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -1667,11 +1667,12 @@ define amdgpu_kernel void @s_log_v3f32(ptr addrspace(1) %out, <3 x float> %in) {
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v4|
; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v11, 0x3377d1cf, v5 :: v_dual_sub_f32 v2, v2, v0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT: v_add_f32_e32 v8, v8, v11
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v5|
-; GFX1100-SDAG-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_sub_f32 v1, v4, v1
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v8, v8, v11
+; GFX1100-SDAG-NEXT: v_sub_f32_e32 v1, v4, v1
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v5, v3
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -2085,33 +2086,33 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; VI-SDAG-NEXT: s_mov_b32 s8, 0x7f800000
; VI-SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s6
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3805fdf4, v5
-; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-SDAG-NEXT: v_mov_b32_e32 v6, s3
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, s6
; VI-SDAG-NEXT: v_ldexp_f32 v6, s2, v6
; VI-SDAG-NEXT: v_log_f32_e32 v6, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x7f800000
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
; VI-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v3, v3, v2
; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v6
@@ -2120,7 +2121,7 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3805fdf4, v2
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_add_f32_e32 v4, v8, v4
; VI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; VI-SDAG-NEXT: v_add_f32_e32 v4, v7, v4
@@ -2130,13 +2131,13 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317000, v2
; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v5
; VI-SDAG-NEXT: v_and_b32_e32 v5, 0xfffff000, v7
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_sub_f32_e32 v6, v7, v5
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3f317000, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v9, 0x3805fdf4, v5
@@ -2149,7 +2150,7 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; VI-SDAG-NEXT: v_log_f32_e32 v6, v1
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v4
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v6
@@ -2161,7 +2162,7 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v0, v4, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v4, s4
@@ -2294,64 +2295,64 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; GFX900-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; GFX900-SDAG-NEXT: s_mov_b32 s8, 0x3f317217
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x7f800000
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s4
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; GFX900-SDAG-NEXT: v_log_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3f317217
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317217, v3
-; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX900-SDAG-NEXT: s_mov_b32 s9, 0x3377d1cf
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s8, -v5
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s3
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s9, v6
-; GFX900-SDAG-NEXT: s_mov_b32 s10, 0x7f800000
+; GFX900-SDAG-NEXT: s_cselect_b32 s8, 32, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3377d1cf
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s3, -v5
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s8
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s4, v6
; GFX900-SDAG-NEXT: v_ldexp_f32 v7, s2, v7
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
; GFX900-SDAG-NEXT: v_log_f32_e32 v7, v7
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317217, v7
; GFX900-SDAG-NEXT: v_mov_b32_e32 v8, s2
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s8, -v2
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s3, -v2
; GFX900-SDAG-NEXT: v_ldexp_f32 v8, s1, v8
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s9, v5
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s4, v5
; GFX900-SDAG-NEXT: v_log_f32_e32 v8, v8
; GFX900-SDAG-NEXT: v_add_f32_e32 v2, v2, v5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v2, v2, v6
; GFX900-SDAG-NEXT: v_mul_f32_e32 v6, 0x3f317217, v8
; GFX900-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s8, -v6
+; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s3, -v6
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s9, v7
+; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s4, v7
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; GFX900-SDAG-NEXT: v_add_f32_e32 v6, v6, v7
; GFX900-SDAG-NEXT: v_log_f32_e32 v7, v1
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v8|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v8|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v5
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317217, v7
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s8, -v5
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s9, v6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s3, -v5
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s4, v6
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v5, v0
@@ -2463,58 +2464,61 @@ define amdgpu_kernel void @s_log_v4f32(ptr addrspace(1) %out, <4 x float> %in) {
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s3
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s8, 0x800000, s1
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s9, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s6
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, s7
-; GFX1100-SDAG-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s3, s6
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 0x41b17218, s8
-; GFX1100-SDAG-NEXT: s_cselect_b32 s7, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s9, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s7, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s3, s9
; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s9, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s7
-; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v6, s1, s3
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v7, s0, s2
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s3
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s8, 0
; GFX1100-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
; GFX1100-SDAG-NEXT: v_log_f32_e32 v3, v3
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v6, s1, s2
+; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, s7
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v7, s0, s1
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1100-SDAG-NEXT: v_log_f32_e32 v6, v6
-; GFX1100-SDAG-NEXT: v_log_f32_e32 v7, v7
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 0x41b17218, s9
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
-; GFX1100-SDAG-NEXT: v_dual_mul_f32 v8, 0x3f317217, v2 :: v_dual_mul_f32 v9, 0x3f317217, v3
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v8, 0x3f317217, v2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
-; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-NEXT: v_dual_mul_f32 v10, 0x3f317217, v6 :: v_dual_mul_f32 v11, 0x3f317217, v7
-; GFX1100-SDAG-NEXT: v_fma_f32 v12, 0x3f317217, v2, -v8
+; GFX1100-SDAG-NEXT: v_log_f32_e32 v7, v7
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v9, 0x3f317217, v3
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 0x41b17218, s8
+; GFX1100-SDAG-NEXT: v_fma_f32 v11, 0x3f317217, v2, -v8
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 0x41b17218, s6
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1100-SDAG-NEXT: v_fma_f32 v13, 0x3f317217, v3, -v9
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT: v_fma_f32 v14, 0x3f317217, v6, -v10
-; GFX1100-SDAG-NEXT: v_fma_f32 v15, 0x3f317217, v7, -v11
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v12, 0x3377d1cf, v2 :: v_dual_fmac_f32 v13, 0x3377d1cf, v3
-; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v14, 0x3377d1cf, v6 :: v_dual_fmac_f32 v15, 0x3377d1cf, v7
+; GFX1100-SDAG-NEXT: v_fmac_f32_e32 v11, 0x3377d1cf, v2
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: v_fmac_f32_e32 v13, 0x3377d1cf, v3
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v10, 0x3f317217, v6
+; GFX1100-SDAG-NEXT: v_dual_add_f32 v8, v8, v11 :: v_dual_add_f32 v9, v9, v13
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_dual_add_f32 v8, v8, v12 :: v_dual_add_f32 v9, v9, v13
-; GFX1100-SDAG-NEXT: v_dual_add_f32 v10, v10, v14 :: v_dual_add_f32 v11, v11, v15
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: v_fma_f32 v14, 0x3f317217, v6, -v10
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v3|
-; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v8, v3, v9 :: v_dual_mov_b32 v9, 0
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v12, 0x3f317217, v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v14, 0x3377d1cf, v6 :: v_dual_cndmask_b32 v9, v3, v9
+; GFX1100-SDAG-NEXT: v_fma_f32 v15, 0x3f317217, v7, -v12
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v6|
+; GFX1100-SDAG-NEXT: v_sub_f32_e32 v3, v2, v0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v10, v10, v14
+; GFX1100-SDAG-NEXT: v_dual_sub_f32 v2, v9, v1 :: v_dual_fmac_f32 v15, 0x3377d1cf, v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc_lo
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v7|
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc_lo
-; GFX1100-SDAG-NEXT: v_dual_sub_f32 v3, v2, v0 :: v_dual_sub_f32 v2, v8, v1
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_dual_sub_f32 v1, v6, v4 :: v_dual_sub_f32 v0, v7, v5
+; GFX1100-SDAG-NEXT: v_dual_add_f32 v8, v12, v15 :: v_dual_sub_f32 v1, v6, v4
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v7, v7, v8 :: v_dual_mov_b32 v8, 0
+; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v7, v5
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX1100-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
; GFX1100-GISEL-LABEL: s_log_v4f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 8fcc16c2e6f56f..ca2bfea10ee64c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -80,19 +80,19 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_log10_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
+; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v2
@@ -154,18 +154,18 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_log10_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3e9a209a
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3284fbcf
@@ -219,23 +219,23 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v1
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v3, 0x3e9a209a, v1, -v2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fmamk_f32 v3, v1, 0x3284fbcf, v3
-; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
@@ -413,19 +413,19 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; VI-SDAG-LABEL: s_log10_contract_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
+; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v2
@@ -487,18 +487,18 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; GFX900-SDAG-LABEL: s_log10_contract_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3e9a209a
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3284fbcf
@@ -552,23 +552,23 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v1
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v3, 0x3e9a209a, v1, -v2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fmamk_f32 v3, v1, 0x3284fbcf, v3
-; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
@@ -787,14 +787,14 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s4
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
@@ -900,38 +900,38 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; GFX900-SDAG-NEXT: s_mov_b32 s6, 0x3284fbcf
-; GFX900-SDAG-NEXT: s_mov_b32 s7, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x7f800000
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, s4
; GFX900-SDAG-NEXT: v_ldexp_f32 v4, s3, v4
; GFX900-SDAG-NEXT: v_log_f32_e32 v4, v4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3e9a209a
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v4
-; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s6, 32, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3284fbcf
; GFX900-SDAG-NEXT: v_fma_f32 v6, v4, s3, -v5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s4
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v4, s6, v6
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v4, s4, v6
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-SDAG-NEXT: v_log_f32_e32 v6, v1
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s7
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v6
; GFX900-SDAG-NEXT: v_fma_f32 v4, v6, s3, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v6, s6, v4
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v6, s4, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s7
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v3, v0
; GFX900-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
@@ -996,33 +996,32 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s4, 0x800000, s3
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s5, 0x800000, s2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s4
-; GFX1100-SDAG-NEXT: s_and_b32 s4, s4, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s4, 32, 0
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, s5
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s5, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s3, s4
-; GFX1100-SDAG-NEXT: s_and_b32 s5, s5, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s5, 32, 0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s5
+; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, s5
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s3
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v3, v3
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v1
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v1|
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v3
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_fma_f32 v6, 0x3e9a209a, v1, -v4
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v7, 0x3e9a209a, v3, -v5
-; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v6, 0x3284fbcf, v1 :: v_dual_fmac_f32 v7, 0x3284fbcf, v3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v6, 0x3284fbcf, v1 :: v_dual_fmac_f32 v7, 0x3284fbcf, v3
; GFX1100-SDAG-NEXT: v_dual_add_f32 v4, v4, v6 :: v_dual_add_f32 v5, v5, v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v4 :: v_dual_mov_b32 v4, 0
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v3|
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_dual_sub_f32 v1, v1, v0 :: v_dual_sub_f32 v0, v3, v2
; GFX1100-SDAG-NEXT: global_store_b64 v4, v[0:1], s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
@@ -1336,37 +1335,37 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
; VI-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; VI-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x369a84fb, v5
-; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; VI-SDAG-NEXT: s_cselect_b32 s5, 32, 0
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-SDAG-NEXT: v_mov_b32_e32 v6, s4
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, s5
; VI-SDAG-NEXT: v_ldexp_f32 v6, s1, v6
; VI-SDAG-NEXT: v_log_f32_e32 v6, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
-; VI-SDAG-NEXT: s_mov_b32 s6, 0x7f800000
+; VI-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
; VI-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s6
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: v_and_b32_e32 v3, 0xfffff000, v6
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_sub_f32_e32 v4, v6, v3
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x369a84fb, v3
@@ -1379,7 +1378,7 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
; VI-SDAG-NEXT: v_log_f32_e32 v4, v1
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s6
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v5
; VI-SDAG-NEXT: v_and_b32_e32 v3, 0xfffff000, v4
@@ -1391,7 +1390,7 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a2000, v3
; VI-SDAG-NEXT: v_add_f32_e32 v3, v3, v5
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s6
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v4|, s4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v0, v3, v0
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -1499,50 +1498,50 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX900-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; GFX900-SDAG-NEXT: s_mov_b32 s8, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s3
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
; GFX900-SDAG-NEXT: v_log_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s5, 32, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3e9a209a
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v3
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s2
-; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x3284fbcf
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s4, -v5
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s5
+; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3284fbcf
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s2, -v5
; GFX900-SDAG-NEXT: v_ldexp_f32 v7, s1, v7
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s5, v6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s3, v6
; GFX900-SDAG-NEXT: v_log_f32_e32 v7, v7
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s8
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v7
; GFX900-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s4, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s2, -v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s5, v5
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s3, v5
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v5
; GFX900-SDAG-NEXT: v_log_f32_e32 v5, v1
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s8
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v6
; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v5
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s4, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s5, v6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s2, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v5, s3, v6
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v6
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s8
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s4
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v3, v0
@@ -1632,34 +1631,35 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s1
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s3
-; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s3, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, s6
-; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s7, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s2, s7
; GFX1100-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s2, s3
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s3
; GFX1100-SDAG-NEXT: v_ldexp_f32 v4, s1, s6
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 0x411a209b, s7
-; GFX1100-SDAG-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s0
; GFX1100-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v4, v4
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 0x411a209b, s3
+; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v5, s0, s2
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_log_f32_e32 v5, v5
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-NEXT: v_dual_mul_f32 v6, 0x3e9a209a, v2 :: v_dual_mul_f32 v7, 0x3e9a209a, v4
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v6, 0x3e9a209a, v2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
+; GFX1100-SDAG-NEXT: v_log_f32_e32 v5, v5
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a209a, v4
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_fma_f32 v9, 0x3e9a209a, v2, -v6
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_fma_f32 v10, 0x3e9a209a, v4, -v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v9, 0x3284fbcf, v2 :: v_dual_fmac_f32 v10, 0x3284fbcf, v4
+; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_mul_f32_e32 v8, 0x3e9a209a, v5
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v7, v7, v10
; GFX1100-SDAG-NEXT: v_fma_f32 v11, 0x3e9a209a, v5, -v8
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -1667,11 +1667,12 @@ define amdgpu_kernel void @s_log10_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v4|
; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v11, 0x3284fbcf, v5 :: v_dual_sub_f32 v2, v2, v0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1100-SDAG-NEXT: v_add_f32_e32 v8, v8, v11
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v5|
-; GFX1100-SDAG-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_sub_f32 v1, v4, v1
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v6, 0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v8, v8, v11
+; GFX1100-SDAG-NEXT: v_sub_f32_e32 v1, v4, v1
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v5, v3
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -2085,33 +2086,33 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; VI-SDAG-NEXT: s_mov_b32 s8, 0x7f800000
; VI-SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s6
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x369a84fb, v5
-; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-SDAG-NEXT: v_mov_b32_e32 v6, s3
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, s6
; VI-SDAG-NEXT: v_ldexp_f32 v6, s2, v6
; VI-SDAG-NEXT: v_log_f32_e32 v6, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x7f800000
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
; VI-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v3, v3, v2
; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v6
@@ -2120,7 +2121,7 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v4
; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x369a84fb, v2
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_add_f32_e32 v4, v8, v4
; VI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; VI-SDAG-NEXT: v_add_f32_e32 v4, v7, v4
@@ -2130,13 +2131,13 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a2000, v2
; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v5
; VI-SDAG-NEXT: v_and_b32_e32 v5, 0xfffff000, v7
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_sub_f32_e32 v6, v7, v5
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3e9a2000, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v6
; VI-SDAG-NEXT: v_mul_f32_e32 v9, 0x369a84fb, v5
@@ -2149,7 +2150,7 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; VI-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; VI-SDAG-NEXT: v_log_f32_e32 v6, v1
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v4
; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v6
@@ -2161,7 +2162,7 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s8
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v6|, s3
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
; VI-SDAG-NEXT: v_sub_f32_e32 v0, v4, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v4, s4
@@ -2294,64 +2295,64 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; GFX900-SDAG-NEXT: s_mov_b32 s8, 0x3e9a209a
+; GFX900-SDAG-NEXT: s_mov_b32 s5, 0x7f800000
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s4
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; GFX900-SDAG-NEXT: v_log_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3e9a209a
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v3
-; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX900-SDAG-NEXT: s_mov_b32 s9, 0x3284fbcf
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s8, -v5
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s3
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s9, v6
-; GFX900-SDAG-NEXT: s_mov_b32 s10, 0x7f800000
+; GFX900-SDAG-NEXT: s_cselect_b32 s8, 32, 0
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x3284fbcf
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s3, -v5
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, s8
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v3, s4, v6
; GFX900-SDAG-NEXT: v_ldexp_f32 v7, s2, v7
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
; GFX900-SDAG-NEXT: v_log_f32_e32 v7, v7
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v7
; GFX900-SDAG-NEXT: v_mov_b32_e32 v8, s2
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s8, -v2
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s3, -v2
; GFX900-SDAG-NEXT: v_ldexp_f32 v8, s1, v8
-; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s9, v5
+; GFX900-SDAG-NEXT: v_fma_f32 v5, v7, s4, v5
; GFX900-SDAG-NEXT: v_log_f32_e32 v8, v8
; GFX900-SDAG-NEXT: v_add_f32_e32 v2, v2, v5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v2, v2, v6
; GFX900-SDAG-NEXT: v_mul_f32_e32 v6, 0x3e9a209a, v8
; GFX900-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s8, -v6
+; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s3, -v6
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s9, v7
+; GFX900-SDAG-NEXT: v_fma_f32 v7, v8, s4, v7
; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s0, v1
; GFX900-SDAG-NEXT: v_add_f32_e32 v6, v6, v7
; GFX900-SDAG-NEXT: v_log_f32_e32 v7, v1
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v8|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v8|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v1, v5
; GFX900-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a209a, v7
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s8, -v5
-; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s9, v6
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s3, -v5
+; GFX900-SDAG-NEXT: v_fma_f32 v6, v7, s4, v6
; GFX900-SDAG-NEXT: v_add_f32_e32 v5, v5, v6
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s10
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v5, v0
@@ -2463,58 +2464,61 @@ define amdgpu_kernel void @s_log10_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s3
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s8, 0x800000, s1
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s9, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s6
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, s7
-; GFX1100-SDAG-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s3, s6
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 0x411a209b, s8
-; GFX1100-SDAG-NEXT: s_cselect_b32 s7, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s9, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s7, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s3, s9
; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s9, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s7
-; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v6, s1, s3
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v7, s0, s2
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s3
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s8, 0
; GFX1100-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
; GFX1100-SDAG-NEXT: v_log_f32_e32 v3, v3
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v6, s1, s2
+; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, s7
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v7, s0, s1
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1100-SDAG-NEXT: v_log_f32_e32 v6, v6
-; GFX1100-SDAG-NEXT: v_log_f32_e32 v7, v7
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 0x411a209b, s9
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
-; GFX1100-SDAG-NEXT: v_dual_mul_f32 v8, 0x3e9a209a, v2 :: v_dual_mul_f32 v9, 0x3e9a209a, v3
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v8, 0x3e9a209a, v2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v2|
-; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-NEXT: v_dual_mul_f32 v10, 0x3e9a209a, v6 :: v_dual_mul_f32 v11, 0x3e9a209a, v7
-; GFX1100-SDAG-NEXT: v_fma_f32 v12, 0x3e9a209a, v2, -v8
+; GFX1100-SDAG-NEXT: v_log_f32_e32 v7, v7
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v9, 0x3e9a209a, v3
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 0x411a209b, s8
+; GFX1100-SDAG-NEXT: v_fma_f32 v11, 0x3e9a209a, v2, -v8
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 0x411a209b, s6
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1100-SDAG-NEXT: v_fma_f32 v13, 0x3e9a209a, v3, -v9
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1100-SDAG-NEXT: v_fma_f32 v14, 0x3e9a209a, v6, -v10
-; GFX1100-SDAG-NEXT: v_fma_f32 v15, 0x3e9a209a, v7, -v11
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v12, 0x3284fbcf, v2 :: v_dual_fmac_f32 v13, 0x3284fbcf, v3
-; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v14, 0x3284fbcf, v6 :: v_dual_fmac_f32 v15, 0x3284fbcf, v7
+; GFX1100-SDAG-NEXT: v_fmac_f32_e32 v11, 0x3284fbcf, v2
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: v_fmac_f32_e32 v13, 0x3284fbcf, v3
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v10, 0x3e9a209a, v6
+; GFX1100-SDAG-NEXT: v_dual_add_f32 v8, v8, v11 :: v_dual_add_f32 v9, v9, v13
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_dual_add_f32 v8, v8, v12 :: v_dual_add_f32 v9, v9, v13
-; GFX1100-SDAG-NEXT: v_dual_add_f32 v10, v10, v14 :: v_dual_add_f32 v11, v11, v15
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: v_fma_f32 v14, 0x3e9a209a, v6, -v10
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v3|
-; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v8, v3, v9 :: v_dual_mov_b32 v9, 0
+; GFX1100-SDAG-NEXT: v_mul_f32_e32 v12, 0x3e9a209a, v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: v_dual_fmac_f32 v14, 0x3284fbcf, v6 :: v_dual_cndmask_b32 v9, v3, v9
+; GFX1100-SDAG-NEXT: v_fma_f32 v15, 0x3e9a209a, v7, -v12
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v6|
+; GFX1100-SDAG-NEXT: v_sub_f32_e32 v3, v2, v0
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: v_add_f32_e32 v10, v10, v14
+; GFX1100-SDAG-NEXT: v_dual_sub_f32 v2, v9, v1 :: v_dual_fmac_f32 v15, 0x3284fbcf, v7
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc_lo
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v7|
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc_lo
-; GFX1100-SDAG-NEXT: v_dual_sub_f32 v3, v2, v0 :: v_dual_sub_f32 v2, v8, v1
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_dual_sub_f32 v1, v6, v4 :: v_dual_sub_f32 v0, v7, v5
+; GFX1100-SDAG-NEXT: v_dual_add_f32 v8, v12, v15 :: v_dual_sub_f32 v1, v6, v4
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v7, v7, v8 :: v_dual_mov_b32 v8, 0
+; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v7, v5
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: global_store_b128 v9, v[0:3], s[0:1]
+; GFX1100-SDAG-NEXT: global_store_b128 v8, v[0:3], s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
; GFX1100-GISEL-LABEL: s_log10_v4f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index fc4ac9040e7e03..39b27a256c5320 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -58,17 +58,17 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_log2_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
; VI-SDAG-NEXT: v_sub_f32_e32 v2, v1, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
@@ -100,18 +100,18 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_log2_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
@@ -145,12 +145,12 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s0, 0x800000, s2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x42000000, s0
-; GFX1100-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s2, s3
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
@@ -294,12 +294,12 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s4
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
@@ -354,12 +354,12 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s4
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
@@ -409,12 +409,11 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s4, 0x800000, s3
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s5, 0x800000, s2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x42000000, s4
-; GFX1100-SDAG-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 0x42000000, s5
; GFX1100-SDAG-NEXT: s_cselect_b32 s4, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s5, s5, exec_lo
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s5, 0
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 0x42000000, s5
; GFX1100-SDAG-NEXT: s_cselect_b32 s5, 32, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s3, s4
; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s5
@@ -610,18 +609,18 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
; VI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v5, s2
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_ldexp_f32 v5, s1, v5
; VI-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
@@ -691,18 +690,18 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v5, s2
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v5, s1, v5
; GFX900-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
@@ -768,24 +767,23 @@ define amdgpu_kernel void @s_log2_v3f32(ptr addrspace(1) %out, <3 x float> %in)
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s1
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x42000000, s3
-; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s0
+; GFX1100-SDAG-NEXT: s_cselect_b32 s7, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, s6
-; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 0x42000000, s7
; GFX1100-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s2, s3
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s2, s7
; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v4, s1, s6
; GFX1100-SDAG-NEXT: v_ldexp_f32 v5, s0, s2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 0x42000000, s3
; GFX1100-SDAG-NEXT: v_log_f32_e32 v2, v2
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v4, v4
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v5, v5
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v6, 0
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -1029,24 +1027,24 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v3, s6
-; VI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
-; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v5, s3
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_ldexp_f32 v5, s2, v5
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
; VI-SDAG-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
; VI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v7, s2
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-NEXT: v_ldexp_f32 v7, s1, v7
; VI-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
@@ -1129,24 +1127,24 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; GFX900-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, s4
-; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
-; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v6, s3
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s1, v0
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v6, s3
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v6, s2, v6
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc
; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
; GFX900-SDAG-NEXT: v_mov_b32_e32 v8, s2
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
+; GFX900-SDAG-NEXT: s_cmp_lg_u64 vcc, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v8, s1, v8
; GFX900-SDAG-NEXT: s_cselect_b32 s1, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
@@ -1225,29 +1223,29 @@ define amdgpu_kernel void @s_log2_v4f32(ptr addrspace(1) %out, <4 x float> %in)
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s3
; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s2
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s8, 0x800000, s1
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s9, 0x800000, s0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x42000000, s6
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s6, 0x800000, s1
; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, s7
-; GFX1100-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 0x42000000, s8
-; GFX1100-SDAG-NEXT: s_cselect_b32 s7, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s3, s6
+; GFX1100-SDAG-NEXT: s_cselect_b32 s8, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s7, 0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s7, 0x800000, s0
+; GFX1100-SDAG-NEXT: s_cselect_b32 s9, 32, 0
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v2, s3, s8
; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_and_b32 s6, s9, exec_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s7
+; GFX1100-SDAG-NEXT: s_cmp_lg_u32 s7, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v3, s2, s9
; GFX1100-SDAG-NEXT: s_cselect_b32 s2, 32, 0
; GFX1100-SDAG-NEXT: v_ldexp_f32 v6, s1, s3
; GFX1100-SDAG-NEXT: v_ldexp_f32 v7, s0, s2
; GFX1100-SDAG-NEXT: v_log_f32_e32 v2, v2
; GFX1100-SDAG-NEXT: v_log_f32_e32 v8, v3
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 0x42000000, s9
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 0x42000000, s6
; GFX1100-SDAG-NEXT: v_log_f32_e32 v6, v6
; GFX1100-SDAG-NEXT: v_log_f32_e32 v7, v7
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 0x42000000, s7
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v9, 0
; GFX1100-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_3)
; GFX1100-SDAG-NEXT: v_dual_sub_f32 v3, v2, v0 :: v_dual_sub_f32 v2, v8, v1
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 5ad9ca723d3fdf..da1c03817f4b7c 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -1198,7 +1198,7 @@ define amdgpu_kernel void @test_umax_ugt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_gt_u64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
@@ -1260,7 +1260,7 @@ define amdgpu_kernel void @test_umax_uge_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_ge_u64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
@@ -1322,7 +1322,7 @@ define amdgpu_kernel void @test_imax_sgt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_gt_i64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
@@ -1384,7 +1384,7 @@ define amdgpu_kernel void @test_imax_sge_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_ge_i64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index cab1b8a45ec5a9..ac4ee03db37256 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -4402,7 +4402,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[1:2]
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, s3, s5
; VI-NEXT: s_cselect_b32 s1, s2, s4
; VI-NEXT: v_mov_b32_e32 v2, s1
@@ -4419,7 +4419,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cselect_b32 s3, s3, s5
; GFX9-NEXT: s_cselect_b32 s2, s2, s4
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -4435,7 +4435,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_lt_u64_e64 s6, s[2:3], s[4:5]
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
; GFX10-NEXT: s_cselect_b32 s2, s2, s4
; GFX10-NEXT: s_cselect_b32 s3, s3, s5
; GFX10-NEXT: v_mov_b32_e32 v0, s2
@@ -4451,7 +4451,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_lt_u64_e64 s6, s[2:3], s[4:5]
-; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_cselect_b32 s2, s2, s4
; GFX11-NEXT: s_cselect_b32 s3, s3, s5
; GFX11-NEXT: v_mov_b32_e32 v0, s2
@@ -4471,7 +4471,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_lt_u64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
@@ -4535,7 +4535,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_cmp_le_u64_e32 vcc, s[2:3], v[1:2]
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, s3, s5
; VI-NEXT: s_cselect_b32 s1, s2, s4
; VI-NEXT: v_mov_b32_e32 v2, s1
@@ -4552,7 +4552,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_le_u64_e32 vcc, s[2:3], v[0:1]
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cselect_b32 s3, s3, s5
; GFX9-NEXT: s_cselect_b32 s2, s2, s4
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -4568,7 +4568,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_le_u64_e64 s6, s[2:3], s[4:5]
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
; GFX10-NEXT: s_cselect_b32 s2, s2, s4
; GFX10-NEXT: s_cselect_b32 s3, s3, s5
; GFX10-NEXT: v_mov_b32_e32 v0, s2
@@ -4584,7 +4584,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_le_u64_e64 s6, s[2:3], s[4:5]
-; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_cselect_b32 s2, s2, s4
; GFX11-NEXT: s_cselect_b32 s3, s3, s5
; GFX11-NEXT: v_mov_b32_e32 v0, s2
@@ -4604,7 +4604,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_le_u64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
@@ -4668,7 +4668,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[1:2]
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, s3, s5
; VI-NEXT: s_cselect_b32 s1, s2, s4
; VI-NEXT: v_mov_b32_e32 v2, s1
@@ -4685,7 +4685,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cselect_b32 s3, s3, s5
; GFX9-NEXT: s_cselect_b32 s2, s2, s4
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -4701,7 +4701,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_lt_i64_e64 s6, s[2:3], s[4:5]
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
; GFX10-NEXT: s_cselect_b32 s2, s2, s4
; GFX10-NEXT: s_cselect_b32 s3, s3, s5
; GFX10-NEXT: v_mov_b32_e32 v0, s2
@@ -4717,7 +4717,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_lt_i64_e64 s6, s[2:3], s[4:5]
-; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_cselect_b32 s2, s2, s4
; GFX11-NEXT: s_cselect_b32 s3, s3, s5
; GFX11-NEXT: v_mov_b32_e32 v0, s2
@@ -4737,7 +4737,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_lt_i64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
@@ -4801,7 +4801,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: v_cmp_le_i64_e32 vcc, s[2:3], v[1:2]
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cmp_lg_u64 vcc, 0
; VI-NEXT: s_cselect_b32 s0, s3, s5
; VI-NEXT: s_cselect_b32 s1, s2, s4
; VI-NEXT: v_mov_b32_e32 v2, s1
@@ -4818,7 +4818,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_le_i64_e32 vcc, s[2:3], v[0:1]
-; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cselect_b32 s3, s3, s5
; GFX9-NEXT: s_cselect_b32 s2, s2, s4
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -4834,7 +4834,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_le_i64_e64 s6, s[2:3], s[4:5]
-; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
; GFX10-NEXT: s_cselect_b32 s2, s2, s4
; GFX10-NEXT: s_cselect_b32 s3, s3, s5
; GFX10-NEXT: v_mov_b32_e32 v0, s2
@@ -4850,7 +4850,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_le_i64_e64 s6, s[2:3], s[4:5]
-; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
; GFX11-NEXT: s_cselect_b32 s2, s2, s4
; GFX11-NEXT: s_cselect_b32 s3, s3, s5
; GFX11-NEXT: v_mov_b32_e32 v0, s2
@@ -4870,7 +4870,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_cmp_le_i64_e64 s4, s[2:3], s[6:7]
-; GFX1250-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
; GFX1250-NEXT: s_cselect_b32 s2, s2, s6
; GFX1250-NEXT: s_cselect_b32 s3, s3, s7
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll b/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
index 7fc67b832a9d82..d77fd76e36a066 100644
--- a/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/mixed-vmem-types.ll
@@ -35,9 +35,8 @@ define amdgpu_cs void @mixed_vmem_types_sampler_reads(i32 inreg %globalTable, i3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s0, s0, s2
; GFX11-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: buffer_store_b32 v0, off, s[24:27], 0
; GFX11-NEXT: s_endpgm
@@ -74,9 +73,8 @@ define amdgpu_cs void @mixed_vmem_types_sampler_reads(i32 inreg %globalTable, i3
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_and_b32 s0, s0, s2
; GFX12-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, s0, exec_lo
; GFX12-NEXT: s_cselect_b32 s0, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: buffer_store_b32 v0, off, s[24:27], null
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/or.ll b/llvm/test/CodeGen/AMDGPU/or.ll
index c079fe82cd8041..635ef6f935ebe2 100644
--- a/llvm/test/CodeGen/AMDGPU/or.ll
+++ b/llvm/test/CodeGen/AMDGPU/or.ll
@@ -910,7 +910,7 @@ define amdgpu_kernel void @or_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, p
; GFX8-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX8-NEXT: v_max_f32_e32 v0, v1, v0
; GFX8-NEXT: v_cmp_le_f32_e32 vcc, 0, v0
-; GFX8-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX8-NEXT: s_cmp_lg_u64 vcc, 0
; GFX8-NEXT: s_cselect_b32 s0, 1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index 8de7a0de592c7e..115c0ba485750e 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -524,16 +524,15 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-SDAG-NEXT: s_xor_b32 s6, s4, 0x80000000
; GFX12-SDAG-NEXT: s_fmac_f32 s5, s6, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_cmp_class_f32_e64 s2, s1, 0x260
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_2)
; GFX12-SDAG-NEXT: s_cmp_gt_f32 s5, 0
-; GFX12-SDAG-NEXT: s_cselect_b32 s2, s4, s3
+; GFX12-SDAG-NEXT: s_cselect_b32 s3, s4, s3
; GFX12-SDAG-NEXT: s_cmp_lt_f32 s0, 0xf800000
-; GFX12-SDAG-NEXT: s_mul_f32 s0, s2, 0x37800000
-; GFX12-SDAG-NEXT: v_cmp_class_f32_e64 s3, s1, 0x260
+; GFX12-SDAG-NEXT: s_mul_f32 s0, s3, 0x37800000
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT: s_cselect_b32 s0, s0, s2
-; GFX12-SDAG-NEXT: s_and_b32 s2, s3, exec_lo
+; GFX12-SDAG-NEXT: s_cselect_b32 s0, s0, s3
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_cselect_b32 s0, s1, s0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f64.ll b/llvm/test/CodeGen/AMDGPU/rsq.f64.ll
index 59374d76a263fe..3ff67920b9f69a 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f64.ll
@@ -205,7 +205,7 @@ define amdgpu_ps <2 x i32> @s_rsq_f64(double inreg %x) {
; VI-SDAG-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-SDAG-CG-NEXT: v_bfrev_b32_e32 v1, 8
; VI-SDAG-CG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; VI-SDAG-CG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-CG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-CG-NEXT: s_cselect_b32 s2, 0x100, 0
; VI-SDAG-CG-NEXT: v_mov_b32_e32 v0, s2
; VI-SDAG-CG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
@@ -776,7 +776,7 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_f64(double inreg %x) {
; VI-SDAG-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-SDAG-CG-NEXT: v_bfrev_b32_e32 v1, 8
; VI-SDAG-CG-NEXT: v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; VI-SDAG-CG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-CG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-CG-NEXT: s_cselect_b32 s2, 0x100, 0
; VI-SDAG-CG-NEXT: v_mov_b32_e32 v0, s2
; VI-SDAG-CG-NEXT: v_ldexp_f64 v[0:1], s[0:1], v0
@@ -1069,7 +1069,7 @@ define amdgpu_ps <2 x i32> @s_neg_rsq_neg_f64(double inreg %x) {
; VI-SDAG-CG-NEXT: v_mov_b32_e32 v0, 0
; VI-SDAG-CG-NEXT: v_bfrev_b32_e32 v1, 9
; VI-SDAG-CG-NEXT: v_cmp_gt_f64_e32 vcc, s[0:1], v[0:1]
-; VI-SDAG-CG-NEXT: s_and_b64 s[2:3], vcc, exec
+; VI-SDAG-CG-NEXT: s_cmp_lg_u64 vcc, 0
; VI-SDAG-CG-NEXT: s_cselect_b32 s2, 0x100, 0
; VI-SDAG-CG-NEXT: v_mov_b32_e32 v0, s2
; VI-SDAG-CG-NEXT: v_ldexp_f64 v[0:1], -s[0:1], v0
diff --git a/llvm/test/CodeGen/AMDGPU/saddo.ll b/llvm/test/CodeGen/AMDGPU/saddo.ll
index 9d8869c765e295..93538a7780d834 100644
--- a/llvm/test/CodeGen/AMDGPU/saddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddo.ll
@@ -299,7 +299,6 @@ define amdgpu_kernel void @v_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; VI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v5
; VI-NEXT: v_cmp_lt_i32_e64 s[0:1], v6, v4
; VI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: flat_store_dword v[0:1], v6
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -546,7 +545,6 @@ define amdgpu_kernel void @v_saddo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; VI-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
; VI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v3
; VI-NEXT: s_xor_b64 s[0:1], s[0:1], vcc
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: flat_store_dwordx2 v[4:5], v[8:9]
@@ -566,7 +564,6 @@ define amdgpu_kernel void @v_saddo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
; GFX9-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v3
; GFX9-NEXT: s_xor_b64 s[0:1], s[0:1], vcc
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]
@@ -587,7 +584,6 @@ define amdgpu_kernel void @v_saddo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
; GFX10-NEXT: s_xor_b32 s0, s0, vcc_lo
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
; GFX10-NEXT: s_cselect_b32 s0, 1, 0
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]
@@ -609,9 +605,8 @@ define amdgpu_kernel void @v_saddo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v3
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
; GFX11-NEXT: s_xor_b32 s0, s0, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b64 v6, v[4:5], s[4:5]
@@ -677,25 +672,24 @@ define amdgpu_kernel void @v_saddo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v3, s7
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: v_mov_b32_e32 v6, s2
; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_u32_e32 v9, vcc, v1, v3
; VI-NEXT: v_add_u32_e32 v8, vcc, v0, v2
; VI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
-; VI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v3
; VI-NEXT: v_cmp_lt_i32_e64 s[2:3], v8, v0
+; VI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v3
; VI-NEXT: v_cmp_lt_i32_e64 s[4:5], v9, v1
; VI-NEXT: s_xor_b64 s[2:3], vcc, s[2:3]
; VI-NEXT: s_xor_b64 s[0:1], s[0:1], s[4:5]
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; VI-NEXT: s_cselect_b32 s4, 1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[2:3], exec
; VI-NEXT: s_cselect_b32 s0, 1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: s_cmp_lg_u64 s[2:3], 0
+; VI-NEXT: s_cselect_b32 s1, 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s1
+; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: flat_store_dwordx2 v[4:5], v[8:9]
; VI-NEXT: flat_store_dwordx2 v[6:7], v[0:1]
; VI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/scmp.ll b/llvm/test/CodeGen/AMDGPU/scmp.ll
index 6ad3fee5344ccc..b98109499544e4 100644
--- a/llvm/test/CodeGen/AMDGPU/scmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/scmp.ll
@@ -669,20 +669,20 @@ define i32 @scmp_i128_uniform(i128 inreg %a, i128 inreg %b) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s4, s[0:1], s[16:17]
; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s5, s[2:3], s[18:19]
-; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s6, s[2:3], s[18:19]
; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, s[0:1], s[16:17]
-; GFX12-SDAG-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s4, s[2:3], s[18:19]
+; GFX12-SDAG-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s5, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s1, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s4, s5, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s5, s6, exec_lo
-; GFX12-SDAG-NEXT: s_cselect_b32 s5, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s0, 1, 0
; GFX12-SDAG-NEXT: s_cmp_eq_u64 s[2:3], s[18:19]
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_cselect_b32 s1, s1, s4
-; GFX12-SDAG-NEXT: s_cselect_b32 s0, s0, s5
+; GFX12-SDAG-NEXT: s_cselect_b32 s1, s6, s1
+; GFX12-SDAG-NEXT: s_cselect_b32 s0, s0, s4
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_and_b32 s1, s1, 1
; GFX12-SDAG-NEXT: s_bitcmp1_b32 s0, 0
@@ -772,9 +772,9 @@ define i32 @scmp_i64_uniform(i64 inreg %a, i64 inreg %b) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s4, s[0:1], s[2:3]
; GFX12-SDAG-NEXT: v_cmp_ge_i64_e64 s0, s[0:1], s[2:3]
-; GFX12-SDAG-NEXT: s_and_b32 s1, s4, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s1, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_cselect_b32 s0, s1, -1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/setcc-f64-hi32mask.ll b/llvm/test/CodeGen/AMDGPU/setcc-f64-hi32mask.ll
index 4ac33651db9277..2bec8839e33a2e 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc-f64-hi32mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc-f64-hi32mask.ll
@@ -56,7 +56,7 @@ define i32 @select.hi32.sgpr.oeq.bad.zero(double inreg %x, double inreg %y, i32
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_eq_f64_e32 vcc, s[16:17], v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT: s_cmp_lg_u64 vcc, 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -79,7 +79,7 @@ define i32 @select.hi32.sgpr.oeq.bad.nan(double inreg %x, double inreg %y, i32 i
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_eq_f64_e32 vcc, s[16:17], v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT: s_cmp_lg_u64 vcc, 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -125,7 +125,7 @@ define i32 @select.hi32.sgpr.ueq.bad.zero(double inreg %x, double inreg %y, i32
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_eq_f64_e32 vcc, s[4:5], v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT: s_cmp_lg_u64 vcc, 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -149,7 +149,7 @@ define i32 @select.hi32.sgpr.ueq.bad.nan(double inreg %x, double inreg %y, i32 i
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_nlg_f64_e32 vcc, s[4:5], v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT: s_cmp_lg_u64 vcc, 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -213,7 +213,7 @@ define i32 @select.hi32.sgpr.olt.bad.neg(double inreg %x, double inreg %y, i32 i
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s4
; CHECK-NEXT: v_cmp_lt_f64_e64 s[4:5], |s[16:17]|, v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -233,7 +233,7 @@ define i32 @select.hi32.sgpr.olt.bad.nan(double inreg %x, double inreg %y, i32 i
; CHECK-NEXT: v_mov_b32_e32 v0, s18
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_lt_f64_e64 s[4:5], |s[16:17]|, |v[0:1]|
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -295,7 +295,7 @@ define i32 @select.hi32.sgpr.ult.bad.0(double inreg %x, double inreg %y, i32 inr
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_nge_f64_e64 s[4:5], |s[16:17]|, v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -317,7 +317,7 @@ define i32 @select.hi32.sgpr.ult.bad.1(double inreg %x, double inreg %y, i32 inr
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_nge_f64_e32 vcc, s[16:17], v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT: s_cmp_lg_u64 vcc, 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -386,7 +386,7 @@ define i32 @select.hi32.sgpr.ole.bad.0(double inreg %x, double inreg %y, i32 inr
; CHECK-NEXT: v_mov_b32_e32 v0, s18
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_le_f64_e32 vcc, s[4:5], v[0:1]
-; CHECK-NEXT: s_and_b64 s[4:5], vcc, exec
+; CHECK-NEXT: s_cmp_lg_u64 vcc, 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -410,7 +410,7 @@ define i32 @select.hi32.sgpr.ole.bad.1(double inreg %x, double inreg %y, i32 inr
; CHECK-NEXT: s_and_b32 s5, s17, 0x3fffffff
; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: v_cmp_le_f64_e64 s[4:5], s[4:5], |v[0:1]|
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
@@ -471,7 +471,7 @@ define i32 @select.hi32.sgpr.ule.bad(double inreg %x, double inreg %y, i32 inreg
; CHECK-NEXT: v_mov_b32_e32 v0, s18
; CHECK-NEXT: v_mov_b32_e32 v1, s19
; CHECK-NEXT: v_cmp_ngt_f64_e64 s[4:5], |s[16:17]|, |v[0:1]|
-; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0
; CHECK-NEXT: s_cselect_b32 s4, s20, s21
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/ssubo.ll b/llvm/test/CodeGen/AMDGPU/ssubo.ll
index 7d08ac6222ecc1..2bb8f842717026 100644
--- a/llvm/test/CodeGen/AMDGPU/ssubo.ll
+++ b/llvm/test/CodeGen/AMDGPU/ssubo.ll
@@ -297,7 +297,6 @@ define amdgpu_kernel void @v_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; VI-NEXT: v_cmp_lt_i32_e32 vcc, v4, v5
; VI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v6
; VI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: flat_store_dword v[0:1], v6
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -546,7 +545,6 @@ define amdgpu_kernel void @v_ssubo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; VI-NEXT: v_subb_u32_e64 v1, s[0:1], v1, v3, s[0:1]
; VI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
; VI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -566,7 +564,6 @@ define amdgpu_kernel void @v_ssubo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v3, s[0:1]
; GFX9-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v1
; GFX9-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[8:9]
; GFX9-NEXT: v_mov_b32_e32 v0, s0
@@ -587,7 +584,6 @@ define amdgpu_kernel void @v_ssubo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
; GFX10-NEXT: v_cmp_gt_i32_e64 s0, 0, v5
; GFX10-NEXT: s_xor_b32 s0, vcc_lo, s0
-; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
; GFX10-NEXT: s_cselect_b32 s0, 1, 0
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]
@@ -609,9 +605,8 @@ define amdgpu_kernel void @v_ssubo_i64(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
; GFX11-NEXT: v_cmp_gt_i32_e64 s0, 0, v5
; GFX11-NEXT: s_xor_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b64 v6, v[4:5], s[4:5]
@@ -683,16 +678,15 @@ define amdgpu_kernel void @v_ssubo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
; VI-NEXT: v_mov_b32_e32 v7, s3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_sub_u32_e32 v8, vcc, v0, v2
-; VI-NEXT: v_sub_u32_e32 v9, vcc, v1, v3
; VI-NEXT: v_cmp_lt_i32_e64 s[0:1], v0, v2
-; VI-NEXT: v_cmp_lt_i32_e32 vcc, v1, v3
+; VI-NEXT: v_sub_u32_e32 v9, vcc, v1, v3
; VI-NEXT: v_cmp_gt_i32_e64 s[2:3], 0, v8
+; VI-NEXT: v_cmp_lt_i32_e32 vcc, v1, v3
; VI-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v9
; VI-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
; VI-NEXT: s_xor_b64 s[2:3], vcc, s[4:5]
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; VI-NEXT: s_cselect_b32 s2, 1, 0
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; VI-NEXT: s_cmp_lg_u64 s[0:1], 0
; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/ucmp.ll b/llvm/test/CodeGen/AMDGPU/ucmp.ll
index 2eef0cd806efb0..4fcc44c05a521f 100644
--- a/llvm/test/CodeGen/AMDGPU/ucmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/ucmp.ll
@@ -664,20 +664,20 @@ define i32 @ucmp_i128_uniform(i128 inreg %a, i128 inreg %b) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s4, s[0:1], s[16:17]
; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s5, s[2:3], s[18:19]
-; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s6, s[2:3], s[18:19]
; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, s[0:1], s[16:17]
-; GFX12-SDAG-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s4, s[2:3], s[18:19]
+; GFX12-SDAG-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s5, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s1, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s4, s5, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s5, s6, exec_lo
-; GFX12-SDAG-NEXT: s_cselect_b32 s5, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s0, 1, 0
; GFX12-SDAG-NEXT: s_cmp_eq_u64 s[2:3], s[18:19]
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-NEXT: s_cselect_b32 s1, s1, s4
-; GFX12-SDAG-NEXT: s_cselect_b32 s0, s0, s5
+; GFX12-SDAG-NEXT: s_cselect_b32 s1, s6, s1
+; GFX12-SDAG-NEXT: s_cselect_b32 s0, s0, s4
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_and_b32 s1, s1, 1
; GFX12-SDAG-NEXT: s_bitcmp1_b32 s0, 0
@@ -767,9 +767,9 @@ define i32 @ucmp_i64_uniform(i64 inreg %a, i64 inreg %b) {
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s4, s[0:1], s[2:3]
; GFX12-SDAG-NEXT: v_cmp_ge_u64_e64 s0, s[0:1], s[2:3]
-; GFX12-SDAG-NEXT: s_and_b32 s1, s4, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-SDAG-NEXT: s_cselect_b32 s1, 1, 0
-; GFX12-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX12-SDAG-NEXT: s_cmp_lg_u32 s0, 0
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-NEXT: s_cselect_b32 s0, s1, -1
; GFX12-SDAG-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll b/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll
index 81a4d41ef7388a..1d6ea7d32f2ac5 100644
--- a/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll
@@ -213,7 +213,7 @@ define amdgpu_ps i64 @s_underflow_compare_fold_i64(i64 inreg %a, i64 inreg %b) #
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
-; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT: s_cmp_lg_u64 vcc, 0
; GFX9-NEXT: s_cselect_b32 s1, s3, s1
; GFX9-NEXT: s_cselect_b32 s0, s2, s0
; GFX9-NEXT: ; return to shader part epilog
@@ -224,7 +224,7 @@ define amdgpu_ps i64 @s_underflow_compare_fold_i64(i64 inreg %a, i64 inreg %b) #
; GFX11-NEXT: s_subb_u32 s3, s1, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_cmp_lt_u64_e64 s4, s[2:3], s[0:1]
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_cselect_b32 s0, s2, s0
; GFX11-NEXT: s_cselect_b32 s1, s3, s1
; GFX11-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index 0e1d873b861709..7df2bec8fb84ed 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -143,7 +143,7 @@ define amdgpu_kernel void @test_vopc_class(ptr addrspace(1) %out, float %x) #0 {
; GFX1032-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_cmp_class_f32_e64 s2, s2, 0x204
-; GFX1032-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
; GFX1032-NEXT: s_cselect_b32 s2, 1, 0
; GFX1032-NEXT: v_mov_b32_e32 v1, s2
; GFX1032-NEXT: global_store_dword v0, v1, s[0:1]
@@ -157,7 +157,7 @@ define amdgpu_kernel void @test_vopc_class(ptr addrspace(1) %out, float %x) #0 {
; GFX1064-NEXT: v_mov_b32_e32 v0, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_cmp_class_f32_e64 s[2:3], s2, 0x204
-; GFX1064-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1064-NEXT: s_cselect_b32 s2, 1, 0
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
; GFX1064-NEXT: global_store_dword v0, v1, s[0:1]
>From 7044c1dcf66370469bc06ac734543d90da090810 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Sat, 5 Sep 2026 14:57:31 +0100
Subject: [PATCH 2/5] Add a TODO
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 37c407f753eeff..26ef4539cd177a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5102,6 +5102,8 @@ static bool isMaskedByExecImpl(const SIInstrInfo &TII,
return true;
if ((Opc == LMC.OrOpc || Opc == LMC.XorOpc) && Recurse(1) && Recurse(2))
return true;
+ // TODO: Sometimes we encounter "reg = S_CSELECT -1, 0". If Reg has no other
+ // uses this could be optimized to "reg = S_CSELECT $exec, 0".
// Roll back, so that a caller that succeeds by another route is not left
// depending on definitions it does not actually use.
>From ffbeffca3a18c53c2141f710d926c2085bd2d716 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at gmail.com>
Date: Sat, 5 Sep 2026 15:01:29 +0100
Subject: [PATCH 3/5] Update
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
index db16640cadbd11..f6db4c91afac77 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-fix-sgpr-copies -o - %s | FileCheck -check-prefixes=CHECK,GFX9 %s
-# RUN: llc -mtriple=amdgcn -mcpu=bonaire -run-pass=si-fix-sgpr-copies -o - %s | FileCheck -check-prefixes=CHECK,GFX7 %s
+# RUN: llc -mtriple=amdgpu9.00 -run-pass=si-fix-sgpr-copies -o - %s | FileCheck -check-prefixes=CHECK,GFX9 %s
+# RUN: llc -mtriple=amdgpu7.00 -run-pass=si-fix-sgpr-copies -o - %s | FileCheck -check-prefixes=CHECK,GFX7 %s
# A copy of a lane mask to SCC is lowered to an AND with EXEC, whose only
# purpose is to define SCC. When the lane mask already has 0 in the bits of
>From 0353570f83654eed6e4fe5b5836e86b703c9b91c Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Mon, 14 Sep 2026 16:00:07 +0100
Subject: [PATCH 4/5] Handle undef use operands
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 7 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
.../AMDGPU/fix-sgpr-copies-scc-cmp.mir | 34 +++++++++++++++++++
3 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index a3869fec8910fc..2907ceb2656b5b 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1213,7 +1213,8 @@ void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
}
if (DstReg == AMDGPU::SCC) {
// Both lowerings below read the whole of SrcReg.
- assert(!MI.getOperand(1).getSubReg() &&
+ const MachineOperand &Src = MI.getOperand(1);
+ assert(!Src.getSubReg() &&
"cannot lower a copy of a subregister to SCC");
MachineBasicBlock::iterator InsPt =
std::next(MachineBasicBlock::iterator(MI));
@@ -1223,14 +1224,14 @@ void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
// needing a destination register.
I = BuildMI(*MI.getParent(), InsPt, MI.getDebugLoc(),
TII->get(LMC.CmpLgOpc))
- .addReg(SrcReg)
+ .add(Src)
.addImm(0);
} else {
Register Tmp = MRI->createVirtualRegister(TRI->getBoolRC());
I = BuildMI(*MI.getParent(), InsPt, MI.getDebugLoc(),
TII->get(LMC.AndOpc))
.addReg(Tmp, getDefRegState(true))
- .addReg(SrcReg)
+ .add(Src)
.addReg(LMC.ExecReg);
}
MI.eraseFromParent();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 26ef4539cd177a..15369a5c4bb5d3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5073,7 +5073,7 @@ static bool isMaskedByExecImpl(const SIInstrInfo &TII,
// themselves, such as V_CMPX, since those would be found by the caller's
// scan for writes to EXEC.
const MachineInstr *Def = MRI.getVRegDef(Reg);
- if (Def->getParent() != MBB)
+ if (!Def || Def->getParent() != MBB)
return false;
size_t NumDefs = Defs.size();
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
index f6db4c91afac77..f3c82c3e797bc1 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
@@ -57,3 +57,37 @@ body: |
%1:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
S_ENDPGM 0, implicit %1
...
+
+# Negative test: a register that is only ever read as undef has no defining
+# instruction at all, so nothing is known about the lane mask.
+---
+name: undef_mask_to_scc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: undef_mask_to_scc
+ ; CHECK: {{%[0-9]+}}:sreg_64 = S_AND_B64 undef %{{[0-9]+}}:sreg_64_xexec, $exec, implicit-def $scc
+ $scc = COPY undef %0:sreg_64_xexec
+ %1:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
+ S_ENDPGM 0, implicit %1
+...
+
+# An undef operand of an AND is not masked by EXEC, but the other operand is,
+# which is enough.
+---
+name: undef_and_operand
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: undef_and_operand
+ ; CHECK: [[CMP:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:sreg_64_xexec = S_AND_B64 undef %{{[0-9]+}}:sreg_64_xexec, [[CMP]], implicit-def dead $scc
+ ; GFX9-NEXT: S_CMP_LG_U64 [[AND]], 0, implicit-def $scc
+ ; GFX7-NEXT: {{%[0-9]+}}:sreg_64 = S_AND_B64 [[AND]], $exec, implicit-def $scc
+ %0:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
+ %2:sreg_64_xexec = S_AND_B64 undef %1:sreg_64_xexec, %0, implicit-def dead $scc
+ $scc = COPY %2
+ %3:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
+ S_ENDPGM 0, implicit %3
+...
>From 49810aeee49c47b51fdac77a2fbc81104d153f12 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Wed, 23 Sep 2026 11:20:29 +0100
Subject: [PATCH 5/5] Remove checks for exec mask modifications
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 64 ++++---------------
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 6 +-
.../AMDGPU/fix-sgpr-copies-scc-cmp.mir | 19 ------
3 files changed, 16 insertions(+), 73 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 709241130e4196..444cfca269cdae 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5136,8 +5136,7 @@ bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
/// Return true if \p MI is a VALU comparison, i.e. an instruction that writes
/// a lane mask with one bit per lane, and zeroes the bits of lanes that were
-/// inactive when it executed. The VOP3 (_e64) encodings are not themselves
-/// marked as VOPC, so also check the VOPC (_e32) equivalent.
+/// inactive when it executed.
///
/// TODO: Also handle the sdst result of V_ADD_CO_U32 and V_SUB_CO_U32 amd
/// V_DIV_SCALE_F32.
@@ -5148,21 +5147,20 @@ static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI) {
return Op32 != -1 && TII.isVOPC(Op32);
}
-/// Worker for SIInstrInfo::isMaskedByExec. Every definition the result depends
-/// on must be in \p MBB, and is added to \p Defs so that the caller can check
-/// that EXEC is not modified between it and the use. Definitions are removed
-/// again when a branch of the search fails, so that on success \p Defs only
-/// holds definitions the result really depends on.
-static bool isMaskedByExecImpl(const SIInstrInfo &TII,
- const AMDGPU::LaneMaskConstants &LMC,
- Register Reg, const MachineBasicBlock *MBB,
- SmallSetVector<const MachineInstr *, 8> &Defs,
- const MachineRegisterInfo &MRI, unsigned Depth) {
+bool SIInstrInfo::isMaskedByExec(Register Reg, const MachineInstr &Use,
+ const MachineRegisterInfo &MRI,
+ unsigned Depth) const {
+ assert(MRI.isSSA() && "isMaskedByExec requires SSA form");
+ const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
+ const MachineBasicBlock *MBB = Use.getParent();
+
// EXEC itself is trivially masked by EXEC.
if (Reg == LMC.ExecReg)
return true;
- if (Depth == 0 || !Reg.isVirtual())
+ // Maximum depth of the def-use walk.
+ constexpr unsigned MaxDepth = 6;
+ if (Depth >= MaxDepth || !Reg.isVirtual())
return false;
// Only look at definitions that can execute under the same EXEC mask as the
@@ -5173,18 +5171,15 @@ static bool isMaskedByExecImpl(const SIInstrInfo &TII,
if (!Def || Def->getParent() != MBB)
return false;
- size_t NumDefs = Defs.size();
- Defs.insert(Def);
-
- if (isVCmp(TII, *Def))
+ if (isVCmp(*this, *Def))
return true;
// Recurse into an operand, which must be a whole register to say anything
- // about the whole lane mask. A failed call rolls back its own additions.
+ // about the whole lane mask.
auto Recurse = [&](unsigned OpIdx) {
const MachineOperand &MO = Def->getOperand(OpIdx);
return MO.isReg() && !MO.getSubReg() &&
- isMaskedByExecImpl(TII, LMC, MO.getReg(), MBB, Defs, MRI, Depth - 1);
+ isMaskedByExec(MO.getReg(), Use, MRI, Depth + 1);
};
unsigned Opc = Def->getOpcode();
@@ -5202,40 +5197,9 @@ static bool isMaskedByExecImpl(const SIInstrInfo &TII,
// TODO: Sometimes we encounter "reg = S_CSELECT -1, 0". If Reg has no other
// uses this could be optimized to "reg = S_CSELECT $exec, 0".
- // Roll back, so that a caller that succeeds by another route is not left
- // depending on definitions it does not actually use.
- while (Defs.size() > NumDefs)
- Defs.pop_back();
return false;
}
-bool SIInstrInfo::isMaskedByExec(Register Reg, const MachineInstr &Use,
- const MachineRegisterInfo &MRI) const {
- assert(MRI.isSSA() && "isMaskedByExec requires SSA form");
- const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
- const MachineBasicBlock *MBB = Use.getParent();
-
- // Maximum depth of the def-use walk.
- constexpr unsigned MaxDepth = 6;
- SmallSetVector<const MachineInstr *, 8> Defs;
- if (!isMaskedByExecImpl(*this, LMC, Reg, MBB, Defs, MRI, MaxDepth))
- return false;
-
- // Walk back from Use, which every definition dominates, and check that all
- // of them are reached before any write to EXEC. Otherwise the mask may have
- // been computed under an EXEC that is not a subset of the one in effect at
- // Use, and so may have bits set for lanes that are now inactive.
- unsigned NumDefsToFind = Defs.size();
- for (const MachineInstr &Prev : reverse(
- make_range(MBB->begin(), MachineBasicBlock::const_iterator(Use)))) {
- if (Prev.modifiesRegister(LMC.ExecReg, &RI))
- return false;
- if (Defs.contains(&Prev) && --NumDefsToFind == 0)
- break;
- }
- return NumDefsToFind == 0;
-}
-
bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
// The src0_modifier operand is present on all instructions
// that have modifiers.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 946626d24d6eba..c0dc46e2a89c60 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1395,11 +1395,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// Return true if \p Reg is a lane mask that already has 0 in every bit
/// corresponding to a lane that is inactive in EXEC where \p Use executes,
- /// so that ANDing it with EXEC there would be a no-op. Only definitions in
- /// the same basic block as \p Use, and not separated from it by a write to
- /// EXEC, are considered. Requires SSA form.
+ /// so that ANDing it with EXEC there would be a no-op. Requires SSA form.
bool isMaskedByExec(Register Reg, const MachineInstr &Use,
- const MachineRegisterInfo &MRI) const;
+ const MachineRegisterInfo &MRI, unsigned Depth = 0) const;
bool physRegUsesConstantBus(const MachineOperand &Reg) const;
bool regUsesConstantBus(const MachineOperand &Reg,
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
index f3c82c3e797bc1..39acda1a0e7085 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-scc-cmp.mir
@@ -39,25 +39,6 @@ body: |
S_ENDPGM 0, implicit %1
...
-# Negative test: EXEC is written between the V_CMP and the copy to SCC, so the
-# V_CMP result may have bits set for lanes that are now inactive.
----
-name: exec_written_in_between
-tracksRegLiveness: true
-body: |
- bb.0:
- liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
- ; CHECK-LABEL: name: exec_written_in_between
- ; CHECK: [[CMP:%[0-9]+]]:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
- ; CHECK-NEXT: $exec = S_OR_B64 $exec, $sgpr0_sgpr1, implicit-def dead $scc
- ; CHECK-NEXT: {{%[0-9]+}}:sreg_64 = S_AND_B64 [[CMP]], $exec, implicit-def $scc
- %0:sreg_64_xexec = V_CMP_LT_I32_e64 $vgpr0, $vgpr1, implicit $exec
- $exec = S_OR_B64 $exec, $sgpr0_sgpr1, implicit-def dead $scc
- $scc = COPY %0
- %1:sreg_32 = S_CSELECT_B32 1, 0, implicit $scc
- S_ENDPGM 0, implicit %1
-...
-
# Negative test: a register that is only ever read as undef has no defining
# instruction at all, so nothing is known about the lane mask.
---
More information about the llvm-commits
mailing list