[llvm] [AMDGPU] Waterfall loop codegen improvement in SIInstrInfo (PR #192415)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 5 04:15:16 PDT 2026
https://github.com/gretay-amd updated https://github.com/llvm/llvm-project/pull/192415
>From 3053495458798607730f06926b9b84cc48ddf08f Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Tue, 31 Mar 2026 15:09:50 +0100
Subject: [PATCH 01/14] [AMDGPU] Waterfall loop codegen improvement
Use the instructions [v_cmpx_eq_*] and [s_andn2_wrexec_*] recommended
for waterfall loops in recent architectures.
This commit only updates code generation in SIInstrInfo.cpp.
Other places that generated waterfall loops can be handled separately.
- Add new lane mask constants for s_andn2_wrexec and v_cmpx_eq_u32/u64
- Fix one .mir test to track liveness needed for verifying phi nodes
- Update .ll and .mir tests to accept the new instruction sequences
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 9 +
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 6 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 139 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 1308 ++++++++--------
.../buffer-fat-pointer-atomicrmw-fmax.ll | 1385 ++++++++---------
.../buffer-fat-pointer-atomicrmw-fmin.ll | 1385 ++++++++---------
...e92561-restore-undef-scc-verifier-error.ll | 69 +-
.../AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll | 28 +-
.../AMDGPU/llvm.amdgcn.dual_intersect_ray.ll | 28 +-
...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll | 74 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 136 +-
...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll | 74 +-
...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll | 146 +-
...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll | 146 +-
...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll | 222 ++-
...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll | 222 ++-
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 80 +-
.../AMDGPU/move-to-valu-vimage-vsample.ll | 84 +-
...uf-legalize-operands-non-ptr-intrinsics.ll | 556 ++++---
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 556 ++++---
.../AMDGPU/mubuf-legalize-operands.mir | 441 ++++--
...r-descriptor-waterfall-loop-idom-update.ll | 53 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 98 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll | 28 +-
24 files changed, 3685 insertions(+), 3588 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index 3fdd962dcc30c..cac303dd22599 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -27,10 +27,13 @@ class LaneMaskConstants {
const unsigned AndN2Opc;
const unsigned AndN2SaveExecOpc;
const unsigned AndN2TermOpc;
+ const unsigned AndN2WRExecOpc;
const unsigned AndSaveExecOpc;
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
const unsigned CMovOpc;
+ const unsigned CmpXEqU32Opc;
+ const unsigned CmpXEqU64Opc;
const unsigned CSelectOpc;
const unsigned MovOpc;
const unsigned MovTermOpc;
@@ -52,12 +55,18 @@ class LaneMaskConstants {
: AMDGPU::S_ANDN2_SAVEEXEC_B64),
AndN2TermOpc(IsWave32 ? AMDGPU::S_ANDN2_B32_term
: AMDGPU::S_ANDN2_B64_term),
+ AndN2WRExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
+ : AMDGPU::S_ANDN2_WREXEC_B64),
AndSaveExecOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64),
AndSaveExecTermOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32_term
: AMDGPU::S_AND_SAVEEXEC_B64_term),
BfmOpc(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64),
CMovOpc(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
+ CmpXEqU32Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32
+ : AMDGPU::V_CMPX_EQ_U32_nosdst_e64),
+ CmpXEqU64Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32
+ : AMDGPU::V_CMPX_EQ_U64_nosdst_e64),
CSelectOpc(IsWave32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64),
MovOpc(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
MovTermOpc(IsWave32 ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term),
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index bcefc3982b4d1..ffa6e60868efb 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -182,6 +182,12 @@ class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
+
+ // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
+ // fail.
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties().setNoPHIs();
+ }
};
} // end anonymous namespace
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 55754ad458eba..125b65fb10a11 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7090,8 +7090,8 @@ void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
// unique value of \p ScalarOps across all lanes. In the best case we execute 1
// iteration, in the worst case we execute 64 (once per lane).
static void emitLoadScalarOpsFromVGPRLoop(
- const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &LoopBB,
- MachineBasicBlock &BodyBB, const DebugLoc &DL,
+ const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
+ MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
MachineFunction &MF = *LoopBB.getParent();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
@@ -7099,8 +7099,33 @@ static void emitLoadScalarOpsFromVGPRLoop(
const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
+ // Emit [v_cmpx_eq] and [s_andn2_wrexec] when these instructions are
+ // available.
+ // Otherwise, use the previous pattern of [v_cmp_eq], [s_and_saveexec],
+ // and [s_xor].
+ // TODO: Accurately detect the availability of [s_andn2_wrexec] instruction
+ // in the target. For now, use the same condition as for the detection
+ // [v_cmpx_eq].
+ bool UseNewExecInstructions = ST.hasNoSdstCMPX();
+
MachineBasicBlock::iterator I = LoopBB.begin();
Register CondReg;
+
+ Register PhiExec = MRI.createVirtualRegister(BoolXExecRC);
+ Register NewExec = MRI.createVirtualRegister(BoolXExecRC);
+
+ if (UseNewExecInstructions) {
+ Register InitExec = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
+ .addReg(LMC.ExecReg);
+
+ BuildMI(LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
+ .addReg(InitExec)
+ .addMBB(&PredBB)
+ .addReg(NewExec)
+ .addMBB(&BodyBB);
+ }
+
for (auto [Idx, ScalarOp] : enumerate(ScalarOps)) {
unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
unsigned NumSubRegs = RegSize / 32;
@@ -7123,21 +7148,27 @@ static void emitLoadScalarOpsFromVGPRLoop(
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
.addReg(VScalarOp);
- Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+ if (UseNewExecInstructions) {
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU32Opc))
+ .addReg(CurReg)
+ .addReg(VScalarOp);
+ } else {
+ Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
- .addReg(CurReg)
- .addReg(VScalarOp);
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
+ .addReg(CurReg)
+ .addReg(VScalarOp);
- // Combine the comparison results with AND.
- if (!CondReg) // First.
- CondReg = NewCondReg;
- else { // If not the first, we create an AND.
- Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
- .addReg(CondReg)
- .addReg(NewCondReg);
- CondReg = AndReg;
+ // Combine the comparison results with AND.
+ if (!CondReg) // First.
+ CondReg = NewCondReg;
+ else { // If not the first, we create an AND.
+ Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ .addReg(CondReg)
+ .addReg(NewCondReg);
+ CondReg = AndReg;
+ }
}
// Update ScalarOp operand to use the SGPR ScalarOp.
@@ -7183,25 +7214,36 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(CurRegHi)
.addImm(AMDGPU::sub1);
- Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
- NewCondReg)
- .addReg(CurReg);
- if (NumSubRegs <= 2)
- Cmp.addReg(VScalarOp);
- else
- Cmp.addReg(VScalarOp, VScalarOpUndef,
- TRI->getSubRegFromChannel(Idx, 2));
-
- // Combine the comparison results with AND.
- if (!CondReg) // First.
- CondReg = NewCondReg;
- else { // If not the first, we create an AND.
- Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
- .addReg(CondReg)
- .addReg(NewCondReg);
- CondReg = AndReg;
+ if (UseNewExecInstructions) {
+ auto CmpX =
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc)).addReg(CurReg);
+
+ if (NumSubRegs <= 2)
+ CmpX.addReg(VScalarOp);
+ else
+ CmpX.addReg(VScalarOp, VScalarOpUndef,
+ TRI->getSubRegFromChannel(Idx, 2));
+ } else {
+ Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+ auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
+ NewCondReg)
+ .addReg(CurReg);
+ if (NumSubRegs <= 2)
+ Cmp.addReg(VScalarOp);
+ else
+ Cmp.addReg(VScalarOp, VScalarOpUndef,
+ TRI->getSubRegFromChannel(Idx, 2));
+
+ // Combine the comparison results with AND.
+ if (!CondReg) // First.
+ CondReg = NewCondReg;
+ else { // If not the first, we create an AND.
+ Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ .addReg(CondReg)
+ .addReg(NewCondReg);
+ CondReg = AndReg;
+ }
}
} // End for loop.
@@ -7230,20 +7272,29 @@ static void emitLoadScalarOpsFromVGPRLoop(
}
}
- Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
- MRI.setSimpleHint(SaveExec, CondReg);
+ Register SaveExec;
+ if (!UseNewExecInstructions) {
+ SaveExec = MRI.createVirtualRegister(BoolXExecRC);
+ MRI.setSimpleHint(SaveExec, CondReg);
- // Update EXEC to matching lanes, saving original to SaveExec.
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
- .addReg(CondReg, RegState::Kill);
+ // Update EXEC to matching lanes, saving original to SaveExec.
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
+ .addReg(CondReg, RegState::Kill);
+ }
// The original instruction is here; we insert the terminators after it.
I = BodyBB.end();
- // Update EXEC, switch all done bits to 0 and all todo bits to 1.
- BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
- .addReg(LMC.ExecReg)
- .addReg(SaveExec);
+ if (UseNewExecInstructions) {
+ MRI.setSimpleHint(NewExec, PhiExec);
+ BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2WRExecOpc), NewExec)
+ .addReg(PhiExec);
+ } else {
+ // Update EXEC, switch all done bits to 0 and all todo bits to 1.
+ BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
+ .addReg(LMC.ExecReg)
+ .addReg(SaveExec);
+ }
BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
}
@@ -7344,7 +7395,7 @@ generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
}
}
- emitLoadScalarOpsFromVGPRLoop(TII, MRI, *LoopBB, *BodyBB, DL, ScalarOps,
+ emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps,
PhySGPRs);
MachineBasicBlock::iterator First = RemainderBB->begin();
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index b5844049fd287..f27ef068911c8 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -372,31 +372,32 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -431,27 +432,25 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -461,60 +460,60 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB2_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f32_e32 v7, v8, v5
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB2_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -2310,66 +2309,66 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB10_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_add_f64_e32 v[11:12], v[13:14], v[5:6]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB10_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB10_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2409,65 +2408,61 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB10_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB10_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB10_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2477,63 +2472,63 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_mov_b32_e32 v7, v2
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v9
-; GFX10-NEXT: v_readfirstlane_b32 s9, v10
-; GFX10-NEXT: v_readfirstlane_b32 s10, v7
-; GFX10-NEXT: v_readfirstlane_b32 s11, v8
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[9:10]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[7:8]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v7
+; GFX10-NEXT: v_readfirstlane_b32 s7, v8
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
+; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[4:7], 0 offen offset:2048
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB10_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v0, v11
; GFX10-NEXT: v_mov_b32_e32 v1, v12
; GFX10-NEXT: v_mov_b32_e32 v2, v13
; GFX10-NEXT: v_mov_b32_e32 v3, v14
; GFX10-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v9
-; GFX10-NEXT: v_readfirstlane_b32 s9, v10
-; GFX10-NEXT: v_readfirstlane_b32 s10, v7
-; GFX10-NEXT: v_readfirstlane_b32 s11, v8
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[9:10]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[7:8]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v7
+; GFX10-NEXT: v_readfirstlane_b32 s7, v8
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB10_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX10-NEXT: v_mov_b32_e32 v14, v1
; GFX10-NEXT: v_mov_b32_e32 v13, v0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB10_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -4193,7 +4188,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4202,67 +4199,65 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4274,7 +4269,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4283,68 +4280,65 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v11, v7
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_add_f16_e32 v6, v6, v5
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4420,8 +4414,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4430,21 +4425,18 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -4452,43 +4444,41 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4496,8 +4486,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4506,29 +4497,27 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v11, v7
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f16_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
@@ -4539,33 +4528,30 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4573,35 +4559,36 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v4, 3, v6
; GFX10-NEXT: v_and_b32_e32 v10, -4, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v11, v7
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_add_f16_e32 v6, v6, v5
; GFX10-NEXT: v_lshlrev_b32_sdwa v6, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v6, v7, v11, v6
@@ -4609,32 +4596,31 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -6015,7 +6001,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6024,80 +6012,77 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6109,7 +6094,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6118,79 +6105,76 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6273,8 +6257,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6283,21 +6268,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
@@ -6306,8 +6288,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
@@ -6328,34 +6311,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6363,8 +6343,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6373,21 +6354,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
@@ -6396,8 +6374,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v10
@@ -6417,34 +6396,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6452,36 +6428,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_add_f32_e32 v4, v4, v10
; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
@@ -6494,32 +6471,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -7346,31 +7322,32 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -7405,124 +7382,120 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b32 v8, v4, s[4:7], 0 offen offset:1024
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b32 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB21_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_add_f16 v7, v8, v5
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v6, v7
; GFX11-NEXT: v_mov_b32_e32 v7, v8
; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-NEXT: v_mov_b32_e32 v8, v6
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: v_mov_b32_e32 v0, v6
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_add_f16 v7, v8, v5
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -9669,31 +9642,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9776,24 +9750,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -9804,8 +9776,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, v6, v8 :: v_dual_add_f32 v5, v5, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -9826,58 +9799,53 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -9888,8 +9856,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v8
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -9910,57 +9879,54 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-FAKE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB28_3: ; %atomicrmw.start
@@ -9969,8 +9935,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_add_f32_e32 v5, v5, v8
; GFX10-NEXT: v_add_f32_e32 v6, v6, v9
; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
@@ -9988,32 +9955,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB28_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 3d85bf7019426..621d53c81345b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -363,31 +363,32 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -453,27 +454,25 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -483,26 +482,26 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: buffer_gl1_inv
@@ -1540,68 +1539,68 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_max_num_f64_e32 v[5:6], v[5:6], v[5:6]
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[11:12], v[0:1], v[5:6]
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB7_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB7_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -1641,25 +1640,23 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
@@ -1667,66 +1664,64 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[11:12], v[0:1], v[5:6]
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB7_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: v_mov_b32_e32 v1, v6
@@ -3312,7 +3307,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3321,70 +3318,67 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v5.l, v5.l
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3396,7 +3390,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3405,71 +3401,69 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v10, v5, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3547,8 +3541,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3557,21 +3552,18 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3580,45 +3572,42 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX11-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3626,8 +3615,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3636,21 +3626,18 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3658,8 +3645,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v10
@@ -3672,33 +3660,30 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3706,36 +3691,37 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
; GFX10-NEXT: v_max_f16_e32 v4, v4, v10
; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
@@ -3744,32 +3730,31 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -5160,7 +5145,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5169,80 +5156,77 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5254,7 +5238,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5263,79 +5249,76 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v4, v10
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5418,8 +5401,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5428,21 +5412,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
@@ -5451,8 +5432,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v10
@@ -5473,34 +5455,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5508,8 +5487,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5518,21 +5498,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
@@ -5541,8 +5518,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v4, v10
@@ -5562,34 +5540,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5597,36 +5572,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f32_e32 v4, v4, v10
; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
@@ -5639,32 +5615,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -6642,67 +6617,67 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v5, v7, v7
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: v_pk_max_num_f16 v6, v5, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mov_b32_e32 v5, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v7
; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-NEXT: v_mov_b32_e32 v7, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -6769,24 +6744,22 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
@@ -6794,103 +6767,101 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_pk_max_f16 v6, v5, v8
; GFX11-NEXT: v_mov_b32_e32 v5, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v7
; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-NEXT: v_mov_b32_e32 v7, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_pk_max_f16 v6, v5, v8
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -8266,40 +8237,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v6, v6, v8 :: v_dual_max_num_f32 v5, v5, v9
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8318,33 +8290,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8355,40 +8325,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v6, v6, v9 :: v_dual_max_num_f32 v5, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8407,33 +8378,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8517,24 +8486,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8545,8 +8512,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_max_f32 v6, v6, v8 :: v_dual_max_f32 v5, v5, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8567,58 +8535,53 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8629,8 +8592,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_max_f32 v6, v6, v9 :: v_dual_max_f32 v5, v5, v8
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8651,57 +8615,54 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
@@ -8710,8 +8671,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f32_e32 v5, v5, v8
; GFX10-NEXT: v_max_f32_e32 v6, v6, v9
; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
@@ -8729,32 +8691,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 3f15d35320573..a8b5c5a201068 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -363,31 +363,32 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -453,27 +454,25 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -483,26 +482,26 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: buffer_gl1_inv
@@ -1540,68 +1539,68 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_max_num_f64_e32 v[5:6], v[5:6], v[5:6]
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[0:1], v[5:6]
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB7_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB7_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -1641,25 +1640,23 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
@@ -1667,66 +1664,64 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_min_f64 v[11:12], v[0:1], v[5:6]
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB7_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: v_mov_b32_e32 v1, v6
@@ -3312,7 +3307,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3321,70 +3318,67 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v5.l, v5.l
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3396,7 +3390,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3405,71 +3401,69 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v10, v5, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3547,8 +3541,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3557,21 +3552,18 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3580,45 +3572,42 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX11-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3626,8 +3615,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3636,21 +3626,18 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3658,8 +3645,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
; GFX11-FAKE16-NEXT: v_min_f16_e32 v4, v4, v10
@@ -3672,33 +3660,30 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3706,36 +3691,37 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
; GFX10-NEXT: v_min_f16_e32 v4, v4, v10
; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
@@ -3744,32 +3730,31 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -5160,7 +5145,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5169,80 +5156,77 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5254,7 +5238,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5263,79 +5249,76 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v4, v10
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5418,8 +5401,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5428,21 +5412,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
@@ -5451,8 +5432,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v10
@@ -5473,34 +5455,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5508,8 +5487,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5518,21 +5498,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
@@ -5541,8 +5518,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v4, v10
@@ -5562,34 +5540,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5597,36 +5572,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_min_f32_e32 v4, v4, v10
; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
@@ -5639,32 +5615,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -6642,67 +6617,67 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v5, v7, v7
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: v_pk_min_num_f16 v6, v5, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mov_b32_e32 v5, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v7
; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-NEXT: v_mov_b32_e32 v7, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -6769,24 +6744,22 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
@@ -6794,103 +6767,101 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_pk_min_f16 v6, v5, v8
; GFX11-NEXT: v_mov_b32_e32 v5, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v7
; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-NEXT: v_mov_b32_e32 v7, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_pk_min_f16 v6, v5, v8
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -8266,40 +8237,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v6, v6, v8 :: v_dual_min_num_f32 v5, v5, v9
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8318,33 +8290,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8355,40 +8325,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v6, v6, v9 :: v_dual_min_num_f32 v5, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8407,33 +8378,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8517,24 +8486,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8545,8 +8512,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_min_f32 v6, v6, v8 :: v_dual_min_f32 v5, v5, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8567,58 +8535,53 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8629,8 +8592,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_min_f32 v6, v6, v9 :: v_dual_min_f32 v5, v5, v8
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8651,57 +8615,54 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
@@ -8710,8 +8671,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_min_f32_e32 v5, v5, v8
; GFX10-NEXT: v_min_f32_e32 v6, v6, v9
; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
@@ -8729,32 +8691,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index 9644c941cd06c..db6b962b8a443 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -15,50 +15,47 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:16
; SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v8, 0
-; SDAG-NEXT: s_mov_b32 s12, 0
-; SDAG-NEXT: s_mov_b32 s3, exec_lo
+; SDAG-NEXT: s_mov_b32 s8, 0
+; SDAG-NEXT: s_mov_b32 s12, exec_lo
+; SDAG-NEXT: s_mov_b32 s9, s8
+; SDAG-NEXT: s_mov_b32 s10, s8
+; SDAG-NEXT: s_mov_b32 s11, s8
; SDAG-NEXT: s_mov_b32 s13, s12
-; SDAG-NEXT: s_mov_b32 s14, s12
-; SDAG-NEXT: s_mov_b32 s15, s12
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; SDAG-NEXT: v_readfirstlane_b32 s6, v2
-; SDAG-NEXT: v_readfirstlane_b32 s7, v3
-; SDAG-NEXT: v_readfirstlane_b32 s8, v4
-; SDAG-NEXT: v_readfirstlane_b32 s9, v5
-; SDAG-NEXT: v_readfirstlane_b32 s10, v6
-; SDAG-NEXT: v_readfirstlane_b32 s11, v7
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; SDAG-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; SDAG-NEXT: v_cmp_eq_u64_e64 s1, s[8:9], v[4:5]
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; SDAG-NEXT: v_cmp_eq_u64_e64 s2, s[10:11], v[6:7]
-; SDAG-NEXT: s_and_b32 s0, vcc_lo, s0
-; SDAG-NEXT: s_and_b32 s0, s0, s1
-; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; SDAG-NEXT: s_and_b32 s0, s0, s2
-; SDAG-NEXT: s_and_saveexec_b32 s0, s0
-; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; SDAG-NEXT: v_readfirstlane_b32 s3, v3
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; SDAG-NEXT: v_readfirstlane_b32 s4, v4
+; SDAG-NEXT: v_readfirstlane_b32 s5, v5
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; SDAG-NEXT: v_readfirstlane_b32 s6, v6
+; SDAG-NEXT: v_readfirstlane_b32 s7, v7
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; SDAG-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; SDAG-NEXT: s_cbranch_execnz .LBB0_1
; SDAG-NEXT: ; %bb.2:
-; SDAG-NEXT: s_mov_b32 exec_lo, s3
+; SDAG-NEXT: s_mov_b32 exec_lo, s12
; SDAG-NEXT: v_dual_mov_b32 v0, 0x7fc00000 :: v_dual_mov_b32 v1, 1.0
-; SDAG-NEXT: s_mov_b32 s0, s12
-; SDAG-NEXT: s_mov_b32 s1, s12
-; SDAG-NEXT: s_mov_b32 s2, s12
-; SDAG-NEXT: s_mov_b32 s3, s12
-; SDAG-NEXT: s_mov_b32 s4, s12
-; SDAG-NEXT: s_mov_b32 s5, s12
-; SDAG-NEXT: s_mov_b32 s6, s12
-; SDAG-NEXT: s_mov_b32 s7, s12
+; SDAG-NEXT: s_mov_b32 s0, s8
+; SDAG-NEXT: s_mov_b32 s1, s8
+; SDAG-NEXT: s_mov_b32 s2, s8
+; SDAG-NEXT: s_mov_b32 s3, s8
+; SDAG-NEXT: s_mov_b32 s4, s8
+; SDAG-NEXT: s_mov_b32 s5, s8
+; SDAG-NEXT: s_mov_b32 s6, s8
+; SDAG-NEXT: s_mov_b32 s7, s8
; SDAG-NEXT: s_clause 0x2
-; SDAG-NEXT: image_sample_c_lz v0, [v8, v8, v0, v8], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v1, [v8, v1, v8, v8], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v0, [v8, v8, v0, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v1, [v8, v1, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: v_dual_add_f32 v0, v9, v0 :: v_dual_mov_b32 v9, v8
; SDAG-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
index 697bacb912f1e..b0e66b1fd116c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
@@ -152,29 +152,29 @@ define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_vvvvvv(i64 %node_ptr, fl
; GFX12-SDAG-NEXT: v_dual_mov_b32 v22, v4 :: v_dual_mov_b32 v25, v1
; GFX12-SDAG-NEXT: v_dual_mov_b32 v26, v2 :: v_dual_mov_b32 v27, 0
; GFX12-SDAG-NEXT: v_mov_b32_e32 v24, v0
-; GFX12-SDAG-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-SDAG-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT: s_mov_b32 s5, s4
; GFX12-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s4, v10
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s5, v11
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s6, v12
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s7, v13
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v10
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s1, v11
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[10:11]
-; GFX12-SDAG-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[12:13]
-; GFX12-SDAG-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v12
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s3, v13
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[21:23], v[18:20], v28], s[4:7]
+; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[21:23], v[18:20], v28], s[0:3]
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr24_vgpr25
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr26_vgpr27
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr28
-; GFX12-SDAG-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
-; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: global_store_b96 v[14:15], v[21:23], off
; GFX12-SDAG-NEXT: global_store_b96 v[16:17], v[18:20], off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
index 97588281b8c75..63de32213e9ba 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
@@ -157,29 +157,29 @@ define amdgpu_ps <10 x float> @image_bvh_dual_intersect_ray_vvvvvv(i64 %node_ptr
; GFX12-SDAG-NEXT: v_dual_mov_b32 v23, v4 :: v_dual_mov_b32 v22, v3
; GFX12-SDAG-NEXT: v_dual_mov_b32 v29, v2 :: v_dual_mov_b32 v28, v1
; GFX12-SDAG-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v30, 0
-; GFX12-SDAG-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-SDAG-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT: s_mov_b32 s5, s4
; GFX12-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s4, v11
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s5, v12
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s6, v13
-; GFX12-SDAG-NEXT: v_readfirstlane_b32 s7, v14
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v11
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s1, v12
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[11:12]
-; GFX12-SDAG-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[13:14]
-; GFX12-SDAG-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[11:12]
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v13
+; GFX12-SDAG-NEXT: v_readfirstlane_b32 s3, v14
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[13:14]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
-; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[22:24], v[19:21], v[25:26]], s[4:7]
+; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[22:24], v[19:21], v[25:26]], s[0:3]
+; GFX12-SDAG-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr11_vgpr12_vgpr13_vgpr14
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr27_vgpr28
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr29_vgpr30
; GFX12-SDAG-NEXT: ; implicit-def: $vgpr25_vgpr26
-; GFX12-SDAG-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX12-SDAG-NEXT: ; %bb.2:
-; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-SDAG-NEXT: s_mov_b32 exec_lo, s4
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: global_store_b96 v[15:16], v[22:24], off
; GFX12-SDAG-NEXT: global_store_b96 v[17:18], v[19:21], off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index 874b336a929a5..a08929ee34529 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -104,33 +104,34 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v6
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v6
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[4:7], s3 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[0:3], s6 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr6
; GFX1200-NEXT: ; implicit-def: $vgpr5
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
@@ -140,33 +141,30 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_dual_mov_b32 v11, v4 :: v_dual_mov_b32 v10, v3
; GFX1250-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v8, v1
; GFX1250-NEXT: v_add_nc_u32_e32 v1, 0x80, v5
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v6
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[8:9]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[10:11]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v6
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s5, s4
+; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[8:9]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v10
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v11
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[10:11]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[4:7], s3 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s6 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11
; GFX1250-NEXT: ; implicit-def: $vgpr6
; GFX1250-NEXT: ; implicit-def: $vgpr1
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 128
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 24fcb933bf801..0395a94a057d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -12,24 +12,24 @@
define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX10-LABEL: main:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-NEXT: s_mov_b32 s5, s4
; GFX10-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: v_readfirstlane_b32 s6, v2
-; GFX10-NEXT: v_readfirstlane_b32 s7, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX10-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX10-NEXT: s_and_saveexec_b32 s0, s0
-; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s1
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v6
@@ -92,25 +92,24 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX11-TRUE16-LABEL: main:
; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
@@ -119,25 +118,24 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX11-FAKE16-LABEL: main:
; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
@@ -145,27 +143,27 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX12-TRUE16-LABEL: main:
; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], null idxen
+; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
@@ -174,27 +172,27 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX12-FAKE16-LABEL: main:
; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], null idxen
+; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 6921cca5a2394..20c70359c8558 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -40,33 +40,34 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB2_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
%ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret <2 x bfloat> %ret
@@ -80,32 +81,33 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[4:7], s3 idxen offen
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB3_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
%ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index d6e140c14d4cc..8455266c6cd15 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -296,32 +296,33 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[4:7], s3 idxen offen
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -331,33 +332,31 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -470,32 +469,33 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[4:7], s3 idxen offen
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -505,33 +505,31 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index d99f1a4a5b996..3fba31a46125d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -245,33 +245,34 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -281,33 +282,31 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -391,33 +390,34 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -427,33 +427,31 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index 3a6cea74de9e9..ec925fa40721f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -461,51 +461,50 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX10-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s2, s1
; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s6, v3
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX11-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s1, s1
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -516,28 +515,29 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s1, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s2, s1
; GFX1200-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX1200-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s1
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
@@ -547,28 +547,27 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s1, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[4:5]
-; GFX1250-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
@@ -637,60 +636,57 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX10-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_readfirstlane_b32 s7, v7
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, s7, v7
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_b32 s4, s4, s5
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
+; GFX10-NEXT: v_readfirstlane_b32 s10, v7
+; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s7 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: v_readfirstlane_b32 s3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_and_b32 s0, s0, s1
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v3
+; GFX11-NEXT: v_readfirstlane_b32 s3, v4
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 glc
+; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -701,33 +697,34 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
@@ -737,33 +734,30 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s5, s4
+; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index a9f843f4a184a..9ee14d4c59915 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -461,51 +461,50 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX10-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s2, s1
; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s6, v3
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX11-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s1, s1
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -516,28 +515,29 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s1, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s2, s1
; GFX1200-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX1200-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s1
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
@@ -547,28 +547,27 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s1, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[4:5]
-; GFX1250-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
@@ -637,60 +636,57 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX10-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_readfirstlane_b32 s7, v7
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, s7, v7
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_b32 s4, s4, s5
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
+; GFX10-NEXT: v_readfirstlane_b32 s10, v7
+; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s7 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: v_readfirstlane_b32 s3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_and_b32 s0, s0, s1
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v3
+; GFX11-NEXT: v_readfirstlane_b32 s3, v4
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 glc
+; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -701,33 +697,34 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
@@ -737,33 +734,30 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s5, s4
+; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index f64f8a81a256f..41cc904135beb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -9,24 +9,24 @@
define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX10-LABEL: main:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-NEXT: s_mov_b32 s5, s4
; GFX10-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: v_readfirstlane_b32 s6, v2
-; GFX10-NEXT: v_readfirstlane_b32 s7, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX10-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX10-NEXT: s_and_saveexec_b32 s0, s0
-; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s1
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v6
@@ -89,25 +89,24 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
;
; GFX11-TRUE16-LABEL: main:
; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
@@ -116,25 +115,24 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
;
; GFX11-FAKE16-LABEL: main:
; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
index 10d363648e3ae..e003e9d33fc2a 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
@@ -19,37 +19,35 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3, [[COPY3]], %subreg.sub4, [[COPY2]], %subreg.sub5, [[COPY1]], %subreg.sub6, [[COPY]], %subreg.sub7
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.1:
; GFX11-NEXT: successors: %bb.2(0x80000000)
; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %14, %bb.2
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX11-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_LOAD_V1_V2_nsa_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_nsa_gfx11 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -73,37 +71,35 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3, [[COPY3]], %subreg.sub4, [[COPY2]], %subreg.sub5, [[COPY1]], %subreg.sub6, [[COPY]], %subreg.sub7
; GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.1:
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %14, %bb.2
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_LOAD_V1_V2_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_gfx12 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -137,37 +133,35 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.1:
; GFX11-NEXT: successors: %bb.2(0x80000000)
; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX11-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -196,37 +190,35 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.1:
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -260,27 +252,27 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.1:
; GFX11-NEXT: successors: %bb.2(0x80000000)
; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX11-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -309,27 +301,27 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.1:
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index c01ab74c5a909..af2e2bfe89a90 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -38,24 +38,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W32-LABEL: mubuf_vgpr:
; GFX1010_W32: ; %bb.0:
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
@@ -63,24 +63,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-LABEL: mubuf_vgpr:
; GFX1010_W64: ; %bb.0:
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
@@ -88,25 +88,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W32-LABEL: mubuf_vgpr:
; GFX1100_W32: ; %bb.0:
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
@@ -114,25 +113,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-LABEL: mubuf_vgpr:
; GFX1100_W64: ; %bb.0:
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
@@ -293,41 +291,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W32: ; %bb.0: ; %entry
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W32-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -339,41 +337,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W64: ; %bb.0: ; %entry
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W64-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -385,44 +383,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W32: ; %bb.0: ; %entry
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
+; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W32-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -434,44 +429,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W64: ; %bb.0: ; %entry
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
+; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W64-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -767,53 +759,53 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s5, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s5, vcc_lo, s5
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, s5
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1010_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1010_W32-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1010_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W32-NEXT: ; %bb.3: ; %bb1
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s10, s9
; GFX1010_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s9
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -823,53 +815,53 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[6:7], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], s[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1010_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX1010_W64-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1010_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W64-NEXT: ; %bb.3: ; %bb1
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[10:11]
; GFX1010_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[10:11]
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -879,57 +871,55 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[10:11], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s5
+; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1100_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W32-NEXT: ; %bb.3: ; %bb1
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W32-NEXT: s_mov_b32 s2, exec_lo
-; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s4
+; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s2
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: .LBB2_6: ; %bb2
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -939,57 +929,55 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[10:11], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1100_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W64-NEXT: ; %bb.3: ; %bb1
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[8:9], exec
-; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: .LBB2_6: ; %bb2
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index b7947de7d5836..0f70397a9fb8e 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -37,24 +37,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W32-LABEL: mubuf_vgpr:
; GFX1010_W32: ; %bb.0:
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
@@ -62,24 +62,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-LABEL: mubuf_vgpr:
; GFX1010_W64: ; %bb.0:
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
@@ -87,25 +87,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W32-LABEL: mubuf_vgpr:
; GFX1100_W32: ; %bb.0:
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
@@ -113,25 +112,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-LABEL: mubuf_vgpr:
; GFX1100_W64: ; %bb.0:
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
@@ -299,41 +297,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W32: ; %bb.0: ; %entry
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W32-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -345,41 +343,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W64: ; %bb.0: ; %entry
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W64-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -391,44 +389,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W32: ; %bb.0: ; %entry
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
+; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W32-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -440,44 +435,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W64: ; %bb.0: ; %entry
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
+; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W64-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -787,53 +779,53 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s5, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s5, vcc_lo, s5
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, s5
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1010_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1010_W32-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1010_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W32-NEXT: ; %bb.3: ; %bb1
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s10, s9
; GFX1010_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s9
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -843,53 +835,53 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[6:7], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], s[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1010_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX1010_W64-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1010_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W64-NEXT: ; %bb.3: ; %bb1
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[10:11]
; GFX1010_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[10:11]
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -899,57 +891,55 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[10:11], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s5
+; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1100_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W32-NEXT: ; %bb.3: ; %bb1
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W32-NEXT: s_mov_b32 s2, exec_lo
-; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s4
+; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s2
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: .LBB2_6: ; %bb2
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -959,57 +949,55 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[10:11], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1100_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W64-NEXT: ; %bb.3: ; %bb1
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[8:9], exec
-; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: .LBB2_6: ; %bb2
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 6ef1574c148b5..5429461bef033 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -1,9 +1,9 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,ADDR64
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,W64-NO-ADDR64
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,W64-NO-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64-PRE-GFX10,W64-PRE-GFX10-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64-PRE-GFX10,W64-PRE-GFX10-NO-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64
# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W32
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,W64-NO-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64
# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W32
# Test that we correctly legalize VGPR Rsrc operands in MUBUF instructions.
@@ -16,6 +16,7 @@
# TODO: S_XOR_B32_term should be `implicit-def $scc`
---
name: idxen
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -26,6 +27,48 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+
+ ; W64-PRE-GFX10-LABEL: name: idxen
+ ; W64-PRE-GFX10: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .1:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .2:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .3:
+ ; W64-PRE-GFX10-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_IDXEN]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
; W64-LABEL: name: idxen
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -38,27 +81,27 @@ body: |
; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .1:
; W64-NEXT: successors: %bb.2(0x80000000)
; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -79,27 +122,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -123,6 +166,7 @@ body: |
---
name: offen
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -133,6 +177,48 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+
+ ; W64-PRE-GFX10-LABEL: name: offen
+ ; W64-PRE-GFX10: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .1:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .2:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .3:
+ ; W64-PRE-GFX10-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFEN]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
; W64-LABEL: name: offen
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -145,27 +231,27 @@ body: |
; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .1:
; W64-NEXT: successors: %bb.2(0x80000000)
; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -186,27 +272,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -230,6 +316,7 @@ body: |
---
name: bothen
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -240,6 +327,49 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+
+
+ ; W64-PRE-GFX10-LABEL: name: bothen
+ ; W64-PRE-GFX10: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .1:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .2:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .3:
+ ; W64-PRE-GFX10-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_BOTHEN]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
; W64-LABEL: name: bothen
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -252,27 +382,27 @@ body: |
; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .1:
; W64-NEXT: successors: %bb.2(0x80000000)
; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -293,27 +423,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -337,6 +467,7 @@ body: |
---
name: addr64
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -347,28 +478,52 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-LABEL: name: addr64
- ; ADDR64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-NEXT: {{ $}}
- ; ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
- ; ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
- ; ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
- ; ADDR64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
- ; ADDR64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; ADDR64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
- ; ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
- ; ADDR64-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY6]].sub0, [[COPY1]].sub0, 0, implicit $exec
- ; ADDR64-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY6]].sub1, [[COPY1]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
- ; ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
- ; ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
- ; ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
- ; ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
- ; ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+
+ ; W64-PRE-GFX10-LABEL: name: addr64
+ ; W64-PRE-GFX10: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY6]].sub0, [[COPY1]].sub0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY6]].sub1, [[COPY1]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
+ ; W64-LABEL: name: addr64
+ ; W64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; W64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; W64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 822173696
+ ; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; W64-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY6]].sub0, [[COPY1]].sub0, 0, implicit $exec
+ ; W64-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY6]].sub1, [[COPY1]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+ ; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
+ ; W64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
+ ; W64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
;
; W32-LABEL: name: addr64
; W32: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -409,6 +564,7 @@ body: |
---
name: offset
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -419,67 +575,110 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-LABEL: name: offset
- ; ADDR64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-NEXT: {{ $}}
- ; ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
- ; ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
- ; ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
- ; ADDR64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
- ; ADDR64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; ADDR64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
- ; ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
- ; ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]].sub0, %subreg.sub0, [[COPY6]].sub1, %subreg.sub1
- ; ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
- ; ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
- ; ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
- ; ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+
+
+ ; W64-PRE-GFX10-ADDR64-LABEL: name: offset
+ ; W64-PRE-GFX10-ADDR64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]].sub0, %subreg.sub0, [[COPY6]].sub1, %subreg.sub1
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
+ ; W64-PRE-GFX10-ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
;
- ; W64-NO-ADDR64-LABEL: name: offset
- ; W64-NO-ADDR64: successors: %bb.1(0x80000000)
- ; W64-NO-ADDR64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
- ; W64-NO-ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
- ; W64-NO-ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; W64-NO-ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; W64-NO-ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; W64-NO-ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
- ; W64-NO-ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: .1:
- ; W64-NO-ADDR64-NEXT: successors: %bb.2(0x80000000)
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NO-ADDR64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: .2:
- ; W64-NO-ADDR64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NO-ADDR64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
- ; W64-NO-ADDR64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: .3:
- ; W64-NO-ADDR64-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
- ; W64-NO-ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
- ; W64-NO-ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFSET]]
- ; W64-NO-ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ; W64-PRE-GFX10-NO-ADDR64-LABEL: name: offset
+ ; W64-PRE-GFX10-NO-ADDR64: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: .1:
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: .2:
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: .3:
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFSET]]
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
+ ; W64-LABEL: name: offset
+ ; W64: successors: %bb.1(0x80000000)
+ ; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .1:
+ ; W64-NEXT: successors: %bb.2(0x80000000)
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .2:
+ ; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .3:
+ ; W64-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFSET]]
+ ; W64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
;
; W32-LABEL: name: offset
; W32: successors: %bb.1(0x80000000)
@@ -493,27 +692,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index 950a1252a3e06..01051646ced0b 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -11,28 +11,28 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-NEXT: ; Child Loop BB0_2 Depth 2
; GCN-NEXT: v_add_co_u32 v6, vcc_lo, v0, 8
; GCN-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
-; GCN-NEXT: s_mov_b32 s5, exec_lo
+; GCN-NEXT: s_mov_b32 s8, exec_lo
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: flat_load_dwordx2 v[4:5], v[6:7]
; GCN-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_mov_b32 s9, s8
; GCN-NEXT: .LBB0_2: ; Parent Loop BB0_1 Depth=1
; GCN-NEXT: ; => This Inner Loop Header: Depth=2
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_readfirstlane_b32 s8, v2
-; GCN-NEXT: v_readfirstlane_b32 s9, v3
-; GCN-NEXT: v_readfirstlane_b32 s10, v4
-; GCN-NEXT: v_readfirstlane_b32 s11, v5
-; GCN-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[2:3]
-; GCN-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[4:5]
-; GCN-NEXT: s_and_b32 s4, vcc_lo, s4
-; GCN-NEXT: s_and_saveexec_b32 s4, s4
-; GCN-NEXT: buffer_store_dword v0, v0, s[8:11], 0 offen
+; GCN-NEXT: v_readfirstlane_b32 s4, v2
+; GCN-NEXT: v_readfirstlane_b32 s5, v3
+; GCN-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
+; GCN-NEXT: v_readfirstlane_b32 s6, v4
+; GCN-NEXT: v_readfirstlane_b32 s7, v5
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
+; GCN-NEXT: buffer_store_dword v0, v0, s[4:7], 0 offen
+; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GCN-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GCN-NEXT: s_cbranch_execnz .LBB0_2
; GCN-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
-; GCN-NEXT: s_mov_b32 exec_lo, s5
+; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GCN-NEXT: s_mov_b32 exec_lo, s8
; GCN-NEXT: s_mov_b32 vcc_lo, exec_lo
; GCN-NEXT: s_cbranch_vccnz .LBB0_1
; GCN-NEXT: ; %bb.4: ; %DummyReturnBlock
@@ -46,26 +46,25 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB0_2 Depth 2
; GFX11-NEXT: flat_load_b128 v[2:5], v[0:1]
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB0_2: ; Parent Loop BB0_1 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v2
-; GFX11-NEXT: v_readfirstlane_b32 s5, v3
-; GFX11-NEXT: v_readfirstlane_b32 s6, v4
-; GFX11-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_store_b32 v0, v0, s[4:7], 0 offen
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: v_readfirstlane_b32 s1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v4
+; GFX11-NEXT: v_readfirstlane_b32 s3, v5
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX11-NEXT: buffer_store_b32 v0, v0, s[0:3], 0 offen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
; GFX11-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_mov_b32 vcc_lo, exec_lo
; GFX11-NEXT: s_cbranch_vccnz .LBB0_1
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 3bc67562012e5..d7c46d71fd016 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -233,9 +233,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %16:vgpr_32, %bb.0, %4, %bb.9
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %48:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %50:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %52:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %50:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -249,24 +249,24 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %54:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %56:vgpr_32, %bb.4, [[PHI1]], %bb.2
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %41, %bb.4
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI undef %58:vgpr_32, %bb.4, [[PHI1]], %bb.2
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], killed [[PHI4]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], killed [[PHI5]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY6]]
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI5]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI6]]
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI4]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -286,24 +286,24 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.7:
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %58:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %60:vgpr_32, %bb.8, [[COPY4]], %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI7:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %33, %bb.8
+ ; SI-NEXT: [[PHI8:%[0-9]+]]:vreg_64 = PHI undef %60:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[PHI9:%[0-9]+]]:vgpr_32 = PHI undef %62:vgpr_32, %bb.8, [[COPY4]], %bb.6
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], killed [[PHI6]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], killed [[PHI8]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY9]]
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI7]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI9]]
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -314,9 +314,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.10.end:
- ; SI-NEXT: [[PHI8:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
+ ; SI-NEXT: [[PHI10:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
; SI-NEXT: SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI8]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI10]]
; SI-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
main_body:
%cc = icmp sgt i32 %z, 5
@@ -356,8 +356,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %16:vgpr_32, %bb.0, %4, %bb.9
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %49:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %51:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %51:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %53:vgpr_32, %bb.9
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -371,12 +371,12 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %53:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI3]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI3]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI3:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %42, %bb.4
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], killed [[PHI3]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], killed [[PHI4]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
@@ -387,7 +387,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -407,12 +407,12 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.7:
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %34, %bb.8
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], killed [[PHI4]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], killed [[PHI6]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
@@ -423,7 +423,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI5]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -434,9 +434,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.10.end:
- ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
+ ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
; SI-NEXT: SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; SI-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI5]], 0, killed [[COPY4]], 0, 0, implicit $mode, implicit $exec
+ ; SI-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI7]], 0, killed [[COPY4]], 0, 0, implicit $mode, implicit $exec
; SI-NEXT: $vgpr0 = COPY killed [[V_ADD_F32_e64_]]
; SI-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
main_body:
@@ -582,27 +582,24 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: bb.2:
; SI-NEXT: successors: %bb.3(0x80000000)
; SI-NEXT: {{ $}}
+ ; SI-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.1, %38, %bb.6
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; SI-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[V_CMP_EQ_U64_e64_]], killed [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; SI-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[S_AND_B32_]], killed [[V_CMP_EQ_U64_e64_2]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; SI-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; SI-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[S_AND_B32_1]], killed [[V_CMP_EQ_U64_e64_3]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; SI-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_2]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub3, killed [[V_READFIRSTLANE_B32_4]], %subreg.sub4, killed [[V_READFIRSTLANE_B32_5]], %subreg.sub5, killed [[V_READFIRSTLANE_B32_6]], %subreg.sub6, killed [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
@@ -612,30 +609,29 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.5(0x80000000)
; SI-NEXT: {{ $}}
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.3, %55, %bb.5
; SI-NEXT: [[V_READFIRSTLANE_B32_8:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_9:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_8]], %subreg.sub0, [[V_READFIRSTLANE_B32_9]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_4:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE6]], [[GLOBAL_LOAD_DWORDX4_2]].sub0_sub1, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE6]], [[GLOBAL_LOAD_DWORDX4_2]].sub0_sub1, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_10:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub2, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_11:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE7:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_10]], %subreg.sub0, [[V_READFIRSTLANE_B32_11]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_5:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE7]], [[GLOBAL_LOAD_DWORDX4_2]].sub2_sub3, implicit $exec
- ; SI-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[V_CMP_EQ_U64_e64_4]], killed [[V_CMP_EQ_U64_e64_5]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE7]], [[GLOBAL_LOAD_DWORDX4_2]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: [[REG_SEQUENCE8:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_8]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_9]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_10]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_11]], %subreg.sub3
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
; SI-NEXT: successors: %bb.4(0x40000000), %bb.6(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[IMAGE_SAMPLE_V1_V2_nsa_gfx10_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_nsa_gfx10 undef %29:vgpr_32, undef %31:vgpr_32, [[REG_SEQUENCE5]], killed [[REG_SEQUENCE8]], 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.4, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.6:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: $exec_lo = S_MOV_B32 killed [[S_MOV_B32_1]]
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.7:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
index b46f5f5640b66..9e0db1dfa26c5 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
@@ -177,18 +177,19 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_cbranch_execz .LBB3_4
; SI-NEXT: ; %bb.1: ; %else
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB3_2: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_readfirstlane_b32 s5, v5
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB3_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -199,18 +200,19 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_cbranch_execz .LBB3_8
; SI-NEXT: ; %bb.5: ; %if
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB3_6: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: v_readfirstlane_b32 s5, v3
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB3_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -254,17 +256,18 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_cbranch_execz .LBB4_4
; SI-NEXT: ; %bb.1: ; %else
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB4_2: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_readfirstlane_b32 s5, v5
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; SI-NEXT: v_mov_b32_e32 v0, v40
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB4_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -274,17 +277,18 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_cbranch_execz .LBB4_8
; SI-NEXT: ; %bb.5: ; %if
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB4_6: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: v_readfirstlane_b32 s5, v3
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; SI-NEXT: v_mov_b32_e32 v0, v40
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB4_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
>From acda1ad8a612416e94c29eb75d415e8e454bd891 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Mon, 11 May 2026 13:01:35 +0100
Subject: [PATCH 02/14] [AMDGPU] NFC: Rename AndN2WRExecOpc to AndN2WrExecOpc
and add availability comment
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 4 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index cac303dd22599..e03e3b0758b8f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -27,7 +27,7 @@ class LaneMaskConstants {
const unsigned AndN2Opc;
const unsigned AndN2SaveExecOpc;
const unsigned AndN2TermOpc;
- const unsigned AndN2WRExecOpc;
+ const unsigned AndN2WrExecOpc; // GFX10+ (HasNoSdstCMPX) only
const unsigned AndSaveExecOpc;
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
@@ -55,7 +55,7 @@ class LaneMaskConstants {
: AMDGPU::S_ANDN2_SAVEEXEC_B64),
AndN2TermOpc(IsWave32 ? AMDGPU::S_ANDN2_B32_term
: AMDGPU::S_ANDN2_B64_term),
- AndN2WRExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
+ AndN2WrExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
: AMDGPU::S_ANDN2_WREXEC_B64),
AndSaveExecOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64),
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 125b65fb10a11..3d778520598fb 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7287,7 +7287,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
if (UseNewExecInstructions) {
MRI.setSimpleHint(NewExec, PhiExec);
- BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2WRExecOpc), NewExec)
+ BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2WrExecOpc), NewExec)
.addReg(PhiExec);
} else {
// Update EXEC, switch all done bits to 0 and all todo bits to 1.
>From ad93d423f78b66c903d1cca9d4756ccff727e9d6 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Mon, 11 May 2026 13:02:47 +0100
Subject: [PATCH 03/14] [AMDGPU] NFC: Move createVirtualRegister calls inside
the if block
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3d778520598fb..3408508f7aa78 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7111,10 +7111,12 @@ static void emitLoadScalarOpsFromVGPRLoop(
MachineBasicBlock::iterator I = LoopBB.begin();
Register CondReg;
- Register PhiExec = MRI.createVirtualRegister(BoolXExecRC);
- Register NewExec = MRI.createVirtualRegister(BoolXExecRC);
+ Register PhiExec;
+ Register NewExec;
if (UseNewExecInstructions) {
+ PhiExec = MRI.createVirtualRegister(BoolXExecRC);
+ NewExec = MRI.createVirtualRegister(BoolXExecRC);
Register InitExec = MRI.createVirtualRegister(BoolXExecRC);
BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
.addReg(LMC.ExecReg);
>From 36edaf4eef91c0675b542b79cd2ee6ba614ff638 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Mon, 11 May 2026 13:02:53 +0100
Subject: [PATCH 04/14] [AMDGPU] Always use VScalarOpUndef flag and factor out
condition
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 25 +++++++++----------------
1 file changed, 9 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3408508f7aa78..3ec1d3067ef90 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7216,25 +7216,18 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(CurRegHi)
.addImm(AMDGPU::sub1);
+ unsigned SubReg =
+ NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
+
if (UseNewExecInstructions) {
- auto CmpX =
- BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc)).addReg(CurReg);
-
- if (NumSubRegs <= 2)
- CmpX.addReg(VScalarOp);
- else
- CmpX.addReg(VScalarOp, VScalarOpUndef,
- TRI->getSubRegFromChannel(Idx, 2));
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc))
+ .addReg(CurReg)
+ .addReg(VScalarOp, VScalarOpUndef, SubReg);
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
- NewCondReg)
- .addReg(CurReg);
- if (NumSubRegs <= 2)
- Cmp.addReg(VScalarOp);
- else
- Cmp.addReg(VScalarOp, VScalarOpUndef,
- TRI->getSubRegFromChannel(Idx, 2));
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
+ .addReg(CurReg)
+ .addReg(VScalarOp, VScalarOpUndef, SubReg);
// Combine the comparison results with AND.
if (!CondReg) // First.
>From 6362f0688eb88a76bc5e507de84b0504df9675c7 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Fri, 15 May 2026 14:10:38 +0100
Subject: [PATCH 05/14] [AMDGPU] Add _term variants of v_cmpx instructions
Add WrapTerminatorVOPC tablegen class (analogous to WrapTerminatorInst
for SALU) and define _term variants for v_cmpx_eq_u32/u64_nosdst
instructions. Handle them in expandPostRAPseudo, analyzeBranch, and
removeTerminatorBit.
These terminators will be used in waterfall loop codegen to prevent the
register allocator from inserting spill code between exec-modifying
v_cmpx and the loop branch.
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 6 ++
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 17 ++++
llvm/lib/Target/AMDGPU/SIInstructions.td | 24 ++++++
.../Target/AMDGPU/SIOptimizeExecMasking.cpp | 12 +++
.../AMDGPU/move-to-valu-vimage-vsample.ll | 40 +++++-----
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 80 +++++++++----------
.../AMDGPU/mubuf-legalize-operands.mir | 32 ++++----
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 20 ++---
8 files changed, 145 insertions(+), 86 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index e03e3b0758b8f..c319035e5c0fc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -33,7 +33,9 @@ class LaneMaskConstants {
const unsigned BfmOpc;
const unsigned CMovOpc;
const unsigned CmpXEqU32Opc;
+ const unsigned CmpXEqU32TermOpc;
const unsigned CmpXEqU64Opc;
+ const unsigned CmpXEqU64TermOpc;
const unsigned CSelectOpc;
const unsigned MovOpc;
const unsigned MovTermOpc;
@@ -65,8 +67,12 @@ class LaneMaskConstants {
CMovOpc(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
CmpXEqU32Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32
: AMDGPU::V_CMPX_EQ_U32_nosdst_e64),
+ CmpXEqU32TermOpc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term
+ : AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term),
CmpXEqU64Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32
: AMDGPU::V_CMPX_EQ_U64_nosdst_e64),
+ CmpXEqU64TermOpc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term
+ : AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term),
CSelectOpc(IsWave32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64),
MovOpc(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
MovTermOpc(IsWave32 ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term),
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3ec1d3067ef90..8b8fcdb399ebd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2117,6 +2117,19 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32));
break;
+ case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
+ MI.setDesc(get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
+ break;
+ case AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term:
+ MI.setDesc(get(AMDGPU::V_CMPX_EQ_U32_nosdst_e64));
+ break;
+ case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
+ MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
+ break;
+ case AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term:
+ MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e64));
+ break;
+
case AMDGPU::SI_SPILL_S32_TO_VGPR:
MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
break;
@@ -3232,6 +3245,10 @@ bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
case AMDGPU::S_ANDN2_B32_term:
case AMDGPU::S_AND_B32_term:
case AMDGPU::S_AND_SAVEEXEC_B32_term:
+ case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
+ case AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term:
+ case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
+ case AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term:
break;
case AMDGPU::SI_IF:
case AMDGPU::SI_ELSE:
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 1338c286445ed..385c0e38018bc 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -496,6 +496,30 @@ def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>;
def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>;
}
+class WrapTerminatorVALU<InstSI base_inst> : VPseudoInstSI<
+ base_inst.OutOperandList,
+ base_inst.InOperandList> {
+ let Uses = base_inst.Uses;
+ let Defs = base_inst.Defs;
+ let isTerminator = 1;
+ let isConvergent = base_inst.isConvergent;
+ let isCompare = base_inst.isCompare;
+ let hasSideEffects = base_inst.hasSideEffects;
+ let UseNamedOperandTable = base_inst.UseNamedOperandTable;
+ let SchedRW = base_inst.SchedRW;
+ let VOPC = base_inst.VOPC;
+}
+
+let SubtargetPredicate = HasNoSdstCMPX in {
+def V_CMPX_EQ_U32_nosdst_e32_term
+ : WrapTerminatorVALU<V_CMPX_EQ_U32_nosdst_e32>;
+def V_CMPX_EQ_U32_nosdst_e64_term
+ : WrapTerminatorVALU<V_CMPX_EQ_U32_nosdst_e64>;
+def V_CMPX_EQ_U64_nosdst_e32_term
+ : WrapTerminatorVALU<V_CMPX_EQ_U64_nosdst_e32>;
+def V_CMPX_EQ_U64_nosdst_e64_term
+ : WrapTerminatorVALU<V_CMPX_EQ_U64_nosdst_e64>;
+}
def WAVE_BARRIER : SPseudoInstSI<(outs), (ins),
[(int_amdgcn_wave_barrier)]> {
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index 47bc218ed9577..7c8fa130f16e0 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -293,6 +293,18 @@ bool SIOptimizeExecMasking::removeTerminatorBit(MachineInstr &MI) const {
MI.setDesc(TII->get(AMDGPU::S_AND_B32));
return true;
}
+ case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
+ MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
+ return true;
+ case AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term:
+ MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U32_nosdst_e64));
+ return true;
+ case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
+ MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
+ return true;
+ case AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term:
+ MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U64_nosdst_e64));
+ return true;
default:
return false;
}
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
index e003e9d33fc2a..e29dab04ed0b1 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
@@ -28,20 +28,20 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -80,20 +80,20 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -142,20 +142,20 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -199,20 +199,20 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -261,12 +261,12 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -310,12 +310,12 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 0f70397a9fb8e..3883db658e750 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -42,10 +42,10 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -67,10 +67,10 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
@@ -93,10 +93,10 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -118,10 +118,10 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
@@ -302,10 +302,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -319,10 +319,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -348,10 +348,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
@@ -365,10 +365,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
@@ -395,10 +395,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -412,10 +412,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -441,10 +441,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
@@ -458,10 +458,10 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
@@ -785,10 +785,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
@@ -809,10 +809,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -841,10 +841,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
@@ -865,10 +865,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
@@ -898,10 +898,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -923,10 +923,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -956,10 +956,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
@@ -981,10 +981,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 5429461bef033..0fc690a11eed9 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -90,12 +90,12 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -131,12 +131,12 @@ body: |
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -240,12 +240,12 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -281,12 +281,12 @@ body: |
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -391,12 +391,12 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -432,12 +432,12 @@ body: |
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -660,12 +660,12 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -701,12 +701,12 @@ body: |
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index d7c46d71fd016..8352e3948611b 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -255,7 +255,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], killed [[PHI5]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], killed [[PHI5]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
@@ -292,7 +292,7 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], killed [[PHI8]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], killed [[PHI8]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
@@ -376,7 +376,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], killed [[PHI4]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], killed [[PHI4]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
@@ -412,7 +412,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], killed [[PHI6]], implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE3]], killed [[PHI6]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
@@ -586,20 +586,20 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; SI-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; SI-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_2]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub3, killed [[V_READFIRSTLANE_B32_4]], %subreg.sub4, killed [[V_READFIRSTLANE_B32_5]], %subreg.sub5, killed [[V_READFIRSTLANE_B32_6]], %subreg.sub6, killed [[V_READFIRSTLANE_B32_7]], %subreg.sub7
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
@@ -613,12 +613,12 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: [[V_READFIRSTLANE_B32_8:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_9:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_8]], %subreg.sub0, [[V_READFIRSTLANE_B32_9]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE6]], [[GLOBAL_LOAD_DWORDX4_2]].sub0_sub1, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_10:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub2, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_11:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE7:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_10]], %subreg.sub0, [[V_READFIRSTLANE_B32_11]], %subreg.sub1
- ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE7]], [[GLOBAL_LOAD_DWORDX4_2]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: [[REG_SEQUENCE8:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_8]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_9]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_10]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_11]], %subreg.sub3
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE6]], [[GLOBAL_LOAD_DWORDX4_2]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32_term killed [[REG_SEQUENCE7]], [[GLOBAL_LOAD_DWORDX4_2]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
; SI-NEXT: successors: %bb.4(0x40000000), %bb.6(0x40000000)
>From 772e4ba835353a584402a647ac8adc1e2a1e591e Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Fri, 15 May 2026 14:10:51 +0100
Subject: [PATCH 06/14] [AMDGPU] Use v_cmpx _term in waterfall loops
Use v_cmpx _term terminators in waterfall loop codegen to prevent the
register allocator from inserting spill code between exec-modifying
v_cmpx and the loop branch.
Each v_cmpx_term is placed in its own block (created by
splitBlockForTerminator) so that no non-terminator instruction follows
it, preserving the MachineVerifier invariant.
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 72 +++++++++++++++++++-------
1 file changed, 52 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 8b8fcdb399ebd..3263d03ea9209 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7103,14 +7103,37 @@ void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
Copy.addReg(AMDGPU::EXEC, RegState::Implicit);
}
+// When a terminator is emitted into a block, no non-terminator instruction may
+// follow it. Split the block: create a new fall-through block after \p BB for
+// subsequent non-terminator instructions.
+// This is similar to SIWholeQuadMode::splitBlock and could be factored out.
+static MachineBasicBlock *
+splitBlockForTerminator(MachineBasicBlock *BB,
+ MachineDominatorTree *MDT) {
+ MachineFunction &MF = *BB->getParent();
+ MachineBasicBlock *NewBB = MF.CreateMachineBasicBlock();
+ MF.insert(std::next(BB->getIterator()), NewBB);
+ NewBB->transferSuccessorsAndUpdatePHIs(BB);
+ BB->addSuccessor(NewBB);
+ if (MDT) {
+ MDT->addNewBlock(NewBB, BB);
+ for (MachineBasicBlock *Succ : NewBB->successors()) {
+ if (MDT->dominates(BB, Succ))
+ MDT->changeImmediateDominator(Succ, NewBB);
+ }
+ }
+ return NewBB;
+}
+
// Emit the actual waterfall loop, executing the wrapped instruction for each
// unique value of \p ScalarOps across all lanes. In the best case we execute 1
// iteration, in the worst case we execute 64 (once per lane).
static void emitLoadScalarOpsFromVGPRLoop(
const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
- MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
- ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
- MachineFunction &MF = *LoopBB.getParent();
+ MachineBasicBlock *LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
+ ArrayRef<MachineOperand *> ScalarOps, MachineDominatorTree *MDT,
+ ArrayRef<Register> PhySGPRs = {}) {
+ MachineFunction &MF = *LoopBB->getParent();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
@@ -7125,7 +7148,11 @@ static void emitLoadScalarOpsFromVGPRLoop(
// [v_cmpx_eq].
bool UseNewExecInstructions = ST.hasNoSdstCMPX();
- MachineBasicBlock::iterator I = LoopBB.begin();
+ // v_cmpx_term instructions are terminators. Each v_cmpx_term is placed at
+ // the end of LoopBB, then the block is split so subsequent non-terminator
+ // instructions go into the new fall-through block.
+ MachineBasicBlock *LoopTarget = LoopBB;
+ MachineBasicBlock::iterator I = LoopBB->begin();
Register CondReg;
Register PhiExec;
@@ -7138,7 +7165,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
.addReg(LMC.ExecReg);
- BuildMI(LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
+ BuildMI(*LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
.addReg(InitExec)
.addMBB(&PredBB)
.addReg(NewExec)
@@ -7164,17 +7191,19 @@ static void emitLoadScalarOpsFromVGPRLoop(
}
Register CurReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
.addReg(VScalarOp);
if (UseNewExecInstructions) {
- BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU32Opc))
+ BuildMI(*LoopBB, I, DL, TII.get(LMC.CmpXEqU32TermOpc))
.addReg(CurReg)
.addReg(VScalarOp);
+ LoopBB = splitBlockForTerminator(LoopBB, MDT);
+ I = LoopBB->end();
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
.addReg(CurReg)
.addReg(VScalarOp);
@@ -7183,7 +7212,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
CondReg = NewCondReg;
else { // If not the first, we create an AND.
Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ BuildMI(*LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
.addReg(CondReg)
.addReg(NewCondReg);
CondReg = AndReg;
@@ -7214,11 +7243,11 @@ static void emitLoadScalarOpsFromVGPRLoop(
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
// Read the next variant <- also loop target.
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
.addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx));
// Read the next variant <- also loop target.
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
.addReg(VScalarOp, VScalarOpUndef,
TRI->getSubRegFromChannel(Idx + 1));
@@ -7227,7 +7256,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
// Comparison is to be done as 64-bit.
Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
.addReg(CurRegLo)
.addImm(AMDGPU::sub0)
.addReg(CurRegHi)
@@ -7237,12 +7266,14 @@ static void emitLoadScalarOpsFromVGPRLoop(
NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
if (UseNewExecInstructions) {
- BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc))
+ BuildMI(*LoopBB, I, DL, TII.get(LMC.CmpXEqU64TermOpc))
.addReg(CurReg)
.addReg(VScalarOp, VScalarOpUndef, SubReg);
+ LoopBB = splitBlockForTerminator(LoopBB, MDT);
+ I = LoopBB->end();
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
.addReg(CurReg)
.addReg(VScalarOp, VScalarOpUndef, SubReg);
@@ -7251,7 +7282,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
CondReg = NewCondReg;
else { // If not the first, we create an AND.
Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ BuildMI(*LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
.addReg(CondReg)
.addReg(NewCondReg);
CondReg = AndReg;
@@ -7265,7 +7296,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
// Build scalar ScalarOp.
auto Merge =
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
+ BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
unsigned Channel = 0;
for (Register Piece : ReadlanePieces) {
Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++));
@@ -7290,7 +7321,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
MRI.setSimpleHint(SaveExec, CondReg);
// Update EXEC to matching lanes, saving original to SaveExec.
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
+ BuildMI(*LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
.addReg(CondReg, RegState::Kill);
}
@@ -7308,7 +7339,8 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(SaveExec);
}
- BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
+ BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP))
+ .addMBB(LoopTarget);
}
// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
@@ -7407,8 +7439,8 @@ generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
}
}
- emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps,
- PhySGPRs);
+ emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, LoopBB, *BodyBB, DL, ScalarOps,
+ MDT, PhySGPRs);
MachineBasicBlock::iterator First = RemainderBB->begin();
// Restore SCC
>From b76e28d4467e89e82fe033117497ffec4f9f2e79 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Tue, 19 May 2026 14:32:37 +0100
Subject: [PATCH 07/14] Revert "[AMDGPU] Use v_cmpx _term in waterfall loops"
This reverts commit ec9c571714bfededeb03ca2d02efb85c2ddcd343.
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 72 +++++++-------------------
1 file changed, 20 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3263d03ea9209..8b8fcdb399ebd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7103,37 +7103,14 @@ void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
Copy.addReg(AMDGPU::EXEC, RegState::Implicit);
}
-// When a terminator is emitted into a block, no non-terminator instruction may
-// follow it. Split the block: create a new fall-through block after \p BB for
-// subsequent non-terminator instructions.
-// This is similar to SIWholeQuadMode::splitBlock and could be factored out.
-static MachineBasicBlock *
-splitBlockForTerminator(MachineBasicBlock *BB,
- MachineDominatorTree *MDT) {
- MachineFunction &MF = *BB->getParent();
- MachineBasicBlock *NewBB = MF.CreateMachineBasicBlock();
- MF.insert(std::next(BB->getIterator()), NewBB);
- NewBB->transferSuccessorsAndUpdatePHIs(BB);
- BB->addSuccessor(NewBB);
- if (MDT) {
- MDT->addNewBlock(NewBB, BB);
- for (MachineBasicBlock *Succ : NewBB->successors()) {
- if (MDT->dominates(BB, Succ))
- MDT->changeImmediateDominator(Succ, NewBB);
- }
- }
- return NewBB;
-}
-
// Emit the actual waterfall loop, executing the wrapped instruction for each
// unique value of \p ScalarOps across all lanes. In the best case we execute 1
// iteration, in the worst case we execute 64 (once per lane).
static void emitLoadScalarOpsFromVGPRLoop(
const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
- MachineBasicBlock *LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
- ArrayRef<MachineOperand *> ScalarOps, MachineDominatorTree *MDT,
- ArrayRef<Register> PhySGPRs = {}) {
- MachineFunction &MF = *LoopBB->getParent();
+ MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
+ ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
+ MachineFunction &MF = *LoopBB.getParent();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
@@ -7148,11 +7125,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
// [v_cmpx_eq].
bool UseNewExecInstructions = ST.hasNoSdstCMPX();
- // v_cmpx_term instructions are terminators. Each v_cmpx_term is placed at
- // the end of LoopBB, then the block is split so subsequent non-terminator
- // instructions go into the new fall-through block.
- MachineBasicBlock *LoopTarget = LoopBB;
- MachineBasicBlock::iterator I = LoopBB->begin();
+ MachineBasicBlock::iterator I = LoopBB.begin();
Register CondReg;
Register PhiExec;
@@ -7165,7 +7138,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
.addReg(LMC.ExecReg);
- BuildMI(*LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
+ BuildMI(LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
.addReg(InitExec)
.addMBB(&PredBB)
.addReg(NewExec)
@@ -7191,19 +7164,17 @@ static void emitLoadScalarOpsFromVGPRLoop(
}
Register CurReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
.addReg(VScalarOp);
if (UseNewExecInstructions) {
- BuildMI(*LoopBB, I, DL, TII.get(LMC.CmpXEqU32TermOpc))
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU32Opc))
.addReg(CurReg)
.addReg(VScalarOp);
- LoopBB = splitBlockForTerminator(LoopBB, MDT);
- I = LoopBB->end();
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
.addReg(CurReg)
.addReg(VScalarOp);
@@ -7212,7 +7183,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
CondReg = NewCondReg;
else { // If not the first, we create an AND.
Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(*LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
.addReg(CondReg)
.addReg(NewCondReg);
CondReg = AndReg;
@@ -7243,11 +7214,11 @@ static void emitLoadScalarOpsFromVGPRLoop(
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
// Read the next variant <- also loop target.
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
.addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx));
// Read the next variant <- also loop target.
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
.addReg(VScalarOp, VScalarOpUndef,
TRI->getSubRegFromChannel(Idx + 1));
@@ -7256,7 +7227,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
// Comparison is to be done as 64-bit.
Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
.addReg(CurRegLo)
.addImm(AMDGPU::sub0)
.addReg(CurRegHi)
@@ -7266,14 +7237,12 @@ static void emitLoadScalarOpsFromVGPRLoop(
NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
if (UseNewExecInstructions) {
- BuildMI(*LoopBB, I, DL, TII.get(LMC.CmpXEqU64TermOpc))
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc))
.addReg(CurReg)
.addReg(VScalarOp, VScalarOpUndef, SubReg);
- LoopBB = splitBlockForTerminator(LoopBB, MDT);
- I = LoopBB->end();
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
.addReg(CurReg)
.addReg(VScalarOp, VScalarOpUndef, SubReg);
@@ -7282,7 +7251,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
CondReg = NewCondReg;
else { // If not the first, we create an AND.
Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(*LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
.addReg(CondReg)
.addReg(NewCondReg);
CondReg = AndReg;
@@ -7296,7 +7265,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
// Build scalar ScalarOp.
auto Merge =
- BuildMI(*LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
unsigned Channel = 0;
for (Register Piece : ReadlanePieces) {
Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++));
@@ -7321,7 +7290,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
MRI.setSimpleHint(SaveExec, CondReg);
// Update EXEC to matching lanes, saving original to SaveExec.
- BuildMI(*LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
.addReg(CondReg, RegState::Kill);
}
@@ -7339,8 +7308,7 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(SaveExec);
}
- BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP))
- .addMBB(LoopTarget);
+ BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
}
// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
@@ -7439,8 +7407,8 @@ generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
}
}
- emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, LoopBB, *BodyBB, DL, ScalarOps,
- MDT, PhySGPRs);
+ emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps,
+ PhySGPRs);
MachineBasicBlock::iterator First = RemainderBB->begin();
// Restore SCC
>From f24e1f8ffe78a356acfc296a9da24518b6682702 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Fri, 15 May 2026 11:06:14 +0100
Subject: [PATCH 08/14] [AMDGPU] Defer v_cmpx_term after all v_readfirstlane
Emit v_cmpx after all readfirstlane in the waterfall loop.
May increase register pressure.
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 17 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 185 ++++++++--------
.../buffer-fat-pointer-atomicrmw-fmax.ll | 197 ++++++++----------
.../buffer-fat-pointer-atomicrmw-fmin.ll | 197 ++++++++----------
...e92561-restore-undef-scc-verifier-error.ll | 9 +-
.../AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll | 5 +-
.../AMDGPU/llvm.amdgcn.dual_intersect_ray.ll | 5 +-
...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll | 17 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 22 +-
...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll | 18 +-
...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll | 34 ++-
...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll | 34 ++-
...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll | 50 ++---
...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll | 50 ++---
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 12 +-
...uf-legalize-operands-non-ptr-intrinsics.ll | 80 +++----
...r-descriptor-waterfall-loop-idom-update.ll | 8 +-
17 files changed, 436 insertions(+), 504 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 8b8fcdb399ebd..b427c29346340 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7145,6 +7145,13 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addMBB(&BodyBB);
}
+ // Current implementation defers v_cmpx and leaves other instruction
+ // scheduling decisions to later passes, where register pressure is known or
+ // easier to approximate.
+ // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
+ // v_cmpx instructions are inserted at the end of LoopBB.
+ // After the first v_cmpx is emitted, I is updated to point to it
+ // so subsequent non-terminators are inserted before all v_cmpx instructions.
for (auto [Idx, ScalarOp] : enumerate(ScalarOps)) {
unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
unsigned NumSubRegs = RegSize / 32;
@@ -7168,9 +7175,12 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(VScalarOp);
if (UseNewExecInstructions) {
- BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU32Opc))
+ auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
+ TII.get(LMC.CmpXEqU32TermOpc))
.addReg(CurReg)
.addReg(VScalarOp);
+ if (I == LoopBB.end())
+ I = CmpxMI.getInstr()->getIterator();
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
@@ -7237,9 +7247,12 @@ static void emitLoadScalarOpsFromVGPRLoop(
NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
if (UseNewExecInstructions) {
- BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc))
+ auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
+ TII.get(LMC.CmpXEqU64TermOpc))
.addReg(CurReg)
.addReg(VScalarOp, VScalarOpUndef, SubReg);
+ if (I == LoopBB.end())
+ I = CmpxMI.getInstr()->getIterator();
} else {
Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index f27ef068911c8..7df50850f6357 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -379,12 +379,11 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -438,10 +437,10 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
@@ -466,10 +465,10 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -491,10 +490,10 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
@@ -2313,15 +2312,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_readfirstlane_b32 s0, v9
; GFX12-NEXT: v_readfirstlane_b32 s1, v10
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_readfirstlane_b32 s2, v7
; GFX12-NEXT: v_readfirstlane_b32 s3, v8
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
@@ -2346,12 +2344,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s0, v9
; GFX12-NEXT: v_readfirstlane_b32 s1, v10
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_readfirstlane_b32 s2, v7
; GFX12-NEXT: v_readfirstlane_b32 s3, v8
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
@@ -2412,13 +2409,13 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -2441,10 +2438,10 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
@@ -2478,10 +2475,10 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v9
; GFX10-NEXT: v_readfirstlane_b32 s5, v10
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
; GFX10-NEXT: v_readfirstlane_b32 s6, v7
; GFX10-NEXT: v_readfirstlane_b32 s7, v8
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[4:7], 0 offen offset:2048
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -2505,10 +2502,10 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v9
; GFX10-NEXT: v_readfirstlane_b32 s5, v10
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
; GFX10-NEXT: v_readfirstlane_b32 s6, v7
; GFX10-NEXT: v_readfirstlane_b32 s7, v8
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
@@ -4201,12 +4198,11 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
@@ -4235,12 +4231,11 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -4282,12 +4277,11 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
@@ -4316,12 +4310,11 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -4427,10 +4420,10 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -4457,10 +4450,10 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
@@ -4499,10 +4492,10 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -4530,10 +4523,10 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
@@ -4570,10 +4563,10 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -4598,10 +4591,10 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
@@ -6014,12 +6007,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -6060,12 +6052,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -6107,12 +6098,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -6152,12 +6142,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -6270,10 +6259,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -6313,10 +6302,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -6356,10 +6345,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -6398,10 +6387,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -6439,10 +6428,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -6473,10 +6462,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
@@ -7329,12 +7318,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -7388,10 +7376,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b32 v8, v4, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -7414,10 +7402,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
@@ -7448,10 +7436,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -7473,10 +7461,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
@@ -9649,12 +9637,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -9756,10 +9743,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -9801,10 +9788,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -9836,10 +9823,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -9881,10 +9868,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -9916,10 +9903,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -9957,10 +9944,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 621d53c81345b..08e1a59245fd6 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -370,12 +370,11 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -460,10 +459,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
@@ -488,10 +487,10 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
@@ -1543,15 +1542,14 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_readfirstlane_b32 s0, v9
; GFX12-NEXT: v_readfirstlane_b32 s1, v10
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_readfirstlane_b32 s2, v7
; GFX12-NEXT: v_readfirstlane_b32 s3, v8
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
@@ -1578,12 +1576,11 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s0, v9
; GFX12-NEXT: v_readfirstlane_b32 s1, v10
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_readfirstlane_b32 s2, v7
; GFX12-NEXT: v_readfirstlane_b32 s3, v8
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
@@ -1644,13 +1641,13 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -1675,10 +1672,10 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
@@ -1708,10 +1705,10 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
@@ -3320,12 +3317,11 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
@@ -3356,12 +3352,11 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -3403,12 +3398,11 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -3441,12 +3435,11 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -3554,10 +3547,10 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -3586,10 +3579,10 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
@@ -3628,10 +3621,10 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -3662,10 +3655,10 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -3702,10 +3695,10 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -3732,10 +3725,10 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
@@ -5158,12 +5151,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -5204,12 +5196,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -5251,12 +5242,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -5296,12 +5286,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -5414,10 +5403,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -5457,10 +5446,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -5500,10 +5489,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -5542,10 +5531,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -5583,10 +5572,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -5617,10 +5606,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
@@ -6623,12 +6612,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
@@ -6655,12 +6643,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -6750,10 +6737,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -6778,10 +6765,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -6812,10 +6799,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -6839,10 +6826,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
@@ -8243,12 +8230,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
@@ -8292,12 +8278,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -8331,12 +8316,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
@@ -8380,12 +8364,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -8492,10 +8475,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -8537,10 +8520,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -8572,10 +8555,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -8617,10 +8600,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -8652,10 +8635,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -8693,10 +8676,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index a8b5c5a201068..be032802e75d1 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -370,12 +370,11 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -460,10 +459,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
@@ -488,10 +487,10 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
@@ -1543,15 +1542,14 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_readfirstlane_b32 s0, v9
; GFX12-NEXT: v_readfirstlane_b32 s1, v10
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_readfirstlane_b32 s2, v7
; GFX12-NEXT: v_readfirstlane_b32 s3, v8
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
@@ -1578,12 +1576,11 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s0, v9
; GFX12-NEXT: v_readfirstlane_b32 s1, v10
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_readfirstlane_b32 s2, v7
; GFX12-NEXT: v_readfirstlane_b32 s3, v8
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
@@ -1644,13 +1641,13 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -1675,10 +1672,10 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-NEXT: v_readfirstlane_b32 s1, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_readfirstlane_b32 s2, v7
; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
@@ -1708,10 +1705,10 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
@@ -3320,12 +3317,11 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
@@ -3356,12 +3352,11 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -3403,12 +3398,11 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -3441,12 +3435,11 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -3554,10 +3547,10 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -3586,10 +3579,10 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
@@ -3628,10 +3621,10 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -3662,10 +3655,10 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -3702,10 +3695,10 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -3732,10 +3725,10 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
@@ -5158,12 +5151,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -5204,12 +5196,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -5251,12 +5242,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
@@ -5296,12 +5286,11 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
@@ -5414,10 +5403,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -5457,10 +5446,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -5500,10 +5489,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -5542,10 +5531,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
@@ -5583,10 +5572,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -5617,10 +5606,10 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
@@ -6623,12 +6612,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
@@ -6655,12 +6643,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -6750,10 +6737,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -6778,10 +6765,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -6812,10 +6799,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -6839,10 +6826,10 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
@@ -8243,12 +8230,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
@@ -8292,12 +8278,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -8331,12 +8316,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
@@ -8380,12 +8364,11 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
@@ -8492,10 +8475,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -8537,10 +8520,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -8572,10 +8555,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -8617,10 +8600,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
@@ -8652,10 +8635,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -8693,10 +8676,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s6, v2
; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index db6b962b8a443..f1684f2330e22 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -25,17 +25,16 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: v_readfirstlane_b32 s0, v0
; SDAG-NEXT: v_readfirstlane_b32 s1, v1
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; SDAG-NEXT: v_readfirstlane_b32 s2, v2
; SDAG-NEXT: v_readfirstlane_b32 s3, v3
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; SDAG-NEXT: v_readfirstlane_b32 s4, v4
; SDAG-NEXT: v_readfirstlane_b32 s5, v5
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; SDAG-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; SDAG-NEXT: v_readfirstlane_b32 s6, v6
; SDAG-NEXT: v_readfirstlane_b32 s7, v7
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
index b0e66b1fd116c..026bf13dcc454 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bvh8_intersect_ray.ll
@@ -158,12 +158,11 @@ define amdgpu_ps <10 x float> @image_bvh8_intersect_ray_vvvvvv(i64 %node_ptr, fl
; GFX12-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v10
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s1, v11
-; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v12
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s3, v13
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: image_bvh8_intersect_ray v[0:9], [v[24:25], v[26:27], v[21:23], v[18:20], v28], s[0:3]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
index 63de32213e9ba..95be1e6ec904f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
@@ -163,12 +163,11 @@ define amdgpu_ps <10 x float> @image_bvh_dual_intersect_ray_vvvvvv(i64 %node_ptr
; GFX12-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s0, v11
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s1, v12
-; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[11:12]
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s2, v13
; GFX12-SDAG-NEXT: v_readfirstlane_b32 s3, v14
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[11:12]
; GFX12-SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[13:14]
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: image_bvh_dual_intersect_ray v[0:9], [v[27:28], v[29:30], v[22:24], v[19:21], v[25:26]], s[0:3]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index a08929ee34529..cc3bd9706887e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -110,16 +110,13 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v6
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[0:3], s6 offen offset:128 th:TH_ATOMIC_RETURN
@@ -145,15 +142,15 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v8
; GFX1250-NEXT: v_readfirstlane_b32 s1, v9
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[8:9]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v10
; GFX1250-NEXT: v_readfirstlane_b32 s3, v11
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[10:11]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[8:9]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[10:11]
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s6 offen th:TH_ATOMIC_RETURN
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 0395a94a057d2..e035bfbf18fb5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -17,10 +17,10 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX10-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s2, v2
; GFX10-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
@@ -98,10 +98,10 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -124,10 +124,10 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -149,12 +149,11 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
@@ -178,12 +177,11 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX12-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 20c70359c8558..7599cde4eb406 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -46,16 +46,13 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
; GFX1200-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
@@ -87,16 +84,13 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
; GFX1200-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index 8455266c6cd15..4723cabc315e5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -302,16 +302,13 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -339,13 +336,13 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -475,16 +472,13 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen
; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -512,13 +506,13 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen
; GFX1250-NEXT: s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index 3fba31a46125d..ec811326b4cb7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -251,16 +251,13 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
@@ -289,13 +286,13 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
@@ -396,16 +393,13 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
@@ -434,13 +428,13 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index ec925fa40721f..b5b3790c90f69 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -466,10 +466,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v1
; GFX10-NEXT: v_readfirstlane_b32 s5, v2
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX10-NEXT: v_readfirstlane_b32 s6, v3
; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
@@ -492,10 +492,10 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s6, v3
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
@@ -521,12 +521,11 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
@@ -551,13 +550,13 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
@@ -641,12 +640,12 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX10-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v1
; GFX10-NEXT: v_readfirstlane_b32 s5, v2
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX10-NEXT: v_readfirstlane_b32 s6, v3
; GFX10-NEXT: v_readfirstlane_b32 s7, v4
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: v_readfirstlane_b32 s10, v7
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
@@ -670,13 +669,13 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s2, v3
; GFX11-NEXT: v_readfirstlane_b32 s3, v4
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
@@ -703,16 +702,13 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1200-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
@@ -738,15 +734,15 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index 9ee14d4c59915..e19c57c3bd0db 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -466,10 +466,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v1
; GFX10-NEXT: v_readfirstlane_b32 s5, v2
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX10-NEXT: v_readfirstlane_b32 s6, v3
; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
@@ -492,10 +492,10 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s6, v3
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
@@ -521,12 +521,11 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
@@ -551,13 +550,13 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
@@ -641,12 +640,12 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX10-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v1
; GFX10-NEXT: v_readfirstlane_b32 s5, v2
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX10-NEXT: v_readfirstlane_b32 s6, v3
; GFX10-NEXT: v_readfirstlane_b32 s7, v4
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: v_readfirstlane_b32 s10, v7
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
@@ -670,13 +669,13 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s2, v3
; GFX11-NEXT: v_readfirstlane_b32 s3, v4
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
@@ -703,16 +702,13 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1200-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
@@ -738,15 +734,15 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index 41cc904135beb..dc4797ce8bc45 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -14,10 +14,10 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX10-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: v_readfirstlane_b32 s1, v1
-; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_readfirstlane_b32 s2, v2
; GFX10-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
@@ -95,10 +95,10 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -121,10 +121,10 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index af2e2bfe89a90..eff11614c2524 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -43,10 +43,10 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -68,10 +68,10 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
@@ -94,10 +94,10 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -119,10 +119,10 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
@@ -296,10 +296,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -313,10 +313,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -342,10 +342,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
@@ -359,10 +359,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
@@ -389,10 +389,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -406,10 +406,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
@@ -435,10 +435,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
@@ -452,10 +452,10 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
@@ -765,10 +765,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
@@ -789,10 +789,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
@@ -821,10 +821,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
@@ -845,10 +845,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
@@ -878,10 +878,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -903,10 +903,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
@@ -936,10 +936,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
@@ -961,10 +961,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index 01051646ced0b..fa2ea184f2f83 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -21,10 +21,10 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v2
; GCN-NEXT: v_readfirstlane_b32 s5, v3
-; GCN-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GCN-NEXT: v_readfirstlane_b32 s6, v4
; GCN-NEXT: v_readfirstlane_b32 s7, v5
+; GCN-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GCN-NEXT: buffer_store_dword v0, v0, s[4:7], 0 offen
; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
@@ -54,10 +54,10 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
; GFX11-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX11-NEXT: v_readfirstlane_b32 s2, v4
; GFX11-NEXT: v_readfirstlane_b32 s3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
; GFX11-NEXT: buffer_store_b32 v0, v0, s[0:3], 0 offen
; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
>From d134f9a45a352ad5bab203586949b10ed9447bc0 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Thu, 14 May 2026 15:10:02 +0100
Subject: [PATCH 09/14] [AMDGPU] NFC: Add comment about latency vs register
pressure trade-off
Document the design choice of emitting v_cmpx inline with readfirstlane
vs deferring all v_cmpx to the end of the block. Inline emission has
lower register pressure; deferred emission has lower latency.
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index b427c29346340..87ca9946a6fa0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7145,6 +7145,20 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addMBB(&BodyBB);
}
+ // Placement of v_cmpx instructions (when index is longer than 64 bit)
+ // involves a trade-off between register pressure and latency:
+ // (a) Defering all v_cmpx after all v_readfirstlane may increase
+ // register pressure because arguments and results of all
+ // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
+ // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
+ // increase latency by placing v_readfirstlane instructions
+ // immediately before v_cmpx instruction that directly depend on it.
+ ///
+ // Emitting interleaved v_cmpx and v_readfirstlane requires
+ // block splitting because v_cmpx changes EXEC mask and therefore for safety
+ // v_cmpx needs to be treated as terminator until after register allocation
+ // (spill placement) and instruction reordering.
+ //
// Current implementation defers v_cmpx and leaves other instruction
// scheduling decisions to later passes, where register pressure is known or
// easier to approximate.
>From 59e1c50d6528a57993725e2a357bd004db2a2711 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Thu, 14 May 2026 15:10:29 +0100
Subject: [PATCH 10/14] [AMDGPU] NFC: Add comment about why saveexec/wrexec
cannot use _term
Document that s_and_saveexec and s_andn2_wrexec define virtual registers,
so making them terminators is not supported.
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 87ca9946a6fa0..892360c0e6041 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7311,6 +7311,9 @@ static void emitLoadScalarOpsFromVGPRLoop(
}
}
+ // Instructions AndSaveExecOpc and AndN2WrExecOpc that modify EXEC mask
+ // should have isTerminator=1 but terminators that define
+ // virtual registers are not supported.
Register SaveExec;
if (!UseNewExecInstructions) {
SaveExec = MRI.createVirtualRegister(BoolXExecRC);
>From 7feb5780ca019f863b5183fb82901d0af13e504c Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Wed, 20 May 2026 14:56:51 +0100
Subject: [PATCH 11/14] Format
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 892360c0e6041..6db411279181e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7189,10 +7189,10 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(VScalarOp);
if (UseNewExecInstructions) {
- auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
- TII.get(LMC.CmpXEqU32TermOpc))
- .addReg(CurReg)
- .addReg(VScalarOp);
+ auto CmpxMI =
+ BuildMI(LoopBB, LoopBB.end(), DL, TII.get(LMC.CmpXEqU32TermOpc))
+ .addReg(CurReg)
+ .addReg(VScalarOp);
if (I == LoopBB.end())
I = CmpxMI.getInstr()->getIterator();
} else {
@@ -7261,10 +7261,10 @@ static void emitLoadScalarOpsFromVGPRLoop(
NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
if (UseNewExecInstructions) {
- auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
- TII.get(LMC.CmpXEqU64TermOpc))
- .addReg(CurReg)
- .addReg(VScalarOp, VScalarOpUndef, SubReg);
+ auto CmpxMI =
+ BuildMI(LoopBB, LoopBB.end(), DL, TII.get(LMC.CmpXEqU64TermOpc))
+ .addReg(CurReg)
+ .addReg(VScalarOp, VScalarOpUndef, SubReg);
if (I == LoopBB.end())
I = CmpxMI.getInstr()->getIterator();
} else {
>From e0068137147dd7a0528b43f84087cbdce7864794 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Fri, 5 Jun 2026 10:55:44 +0100
Subject: [PATCH 12/14] Bug fix: cmpx does not depend on wave size.
Using shorter encoding is sufficient.
Assisted-by: Claude <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 12 ------
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 6 ++-
...uf-legalize-operands-non-ptr-intrinsics.ll | 40 +++++++++----------
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 40 +++++++++----------
.../AMDGPU/mubuf-legalize-operands.mir | 16 ++++----
5 files changed, 52 insertions(+), 62 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index c319035e5c0fc..af14066c1bd46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -32,10 +32,6 @@ class LaneMaskConstants {
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
const unsigned CMovOpc;
- const unsigned CmpXEqU32Opc;
- const unsigned CmpXEqU32TermOpc;
- const unsigned CmpXEqU64Opc;
- const unsigned CmpXEqU64TermOpc;
const unsigned CSelectOpc;
const unsigned MovOpc;
const unsigned MovTermOpc;
@@ -65,14 +61,6 @@ class LaneMaskConstants {
: AMDGPU::S_AND_SAVEEXEC_B64_term),
BfmOpc(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64),
CMovOpc(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
- CmpXEqU32Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32
- : AMDGPU::V_CMPX_EQ_U32_nosdst_e64),
- CmpXEqU32TermOpc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term
- : AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term),
- CmpXEqU64Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32
- : AMDGPU::V_CMPX_EQ_U64_nosdst_e64),
- CmpXEqU64TermOpc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term
- : AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term),
CSelectOpc(IsWave32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64),
MovOpc(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
MovTermOpc(IsWave32 ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term),
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6db411279181e..ef47adfe2e20a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7190,7 +7190,8 @@ static void emitLoadScalarOpsFromVGPRLoop(
if (UseNewExecInstructions) {
auto CmpxMI =
- BuildMI(LoopBB, LoopBB.end(), DL, TII.get(LMC.CmpXEqU32TermOpc))
+ BuildMI(LoopBB, LoopBB.end(), DL,
+ TII.get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
.addReg(CurReg)
.addReg(VScalarOp);
if (I == LoopBB.end())
@@ -7262,7 +7263,8 @@ static void emitLoadScalarOpsFromVGPRLoop(
if (UseNewExecInstructions) {
auto CmpxMI =
- BuildMI(LoopBB, LoopBB.end(), DL, TII.get(LMC.CmpXEqU64TermOpc))
+ BuildMI(LoopBB, LoopBB.end(), DL,
+ TII.get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
.addReg(CurReg)
.addReg(VScalarOp, VScalarOpUndef, SubReg);
if (I == LoopBB.end())
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index eff11614c2524..04d763f0ea6ea 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -71,8 +71,8 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -122,8 +122,8 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -345,8 +345,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -362,8 +362,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
@@ -438,8 +438,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -455,8 +455,8 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
@@ -824,8 +824,8 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -848,8 +848,8 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
@@ -939,8 +939,8 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -964,8 +964,8 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 3883db658e750..7a60a79f90a90 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -70,8 +70,8 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -121,8 +121,8 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -351,8 +351,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -368,8 +368,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
@@ -444,8 +444,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -461,8 +461,8 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
@@ -844,8 +844,8 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -868,8 +868,8 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
-; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
@@ -959,8 +959,8 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -984,8 +984,8 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
-; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 0fc690a11eed9..e1b31c4ec1a70 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -94,8 +94,8 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -244,8 +244,8 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -395,8 +395,8 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
@@ -664,8 +664,8 @@ body: |
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
- ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e32_term [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
>From cc3bbcd4a87d815d6d95c31d13730527374fa379 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Fri, 5 Jun 2026 11:34:27 +0100
Subject: [PATCH 13/14] Cleanup: remove unused _term definitions introduced
early in this PR
Tighten the type on the definition of the new WrapTerminators.
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 --------
llvm/lib/Target/AMDGPU/SIInstructions.td | 20 +++++++------------
.../Target/AMDGPU/SIOptimizeExecMasking.cpp | 6 ------
3 files changed, 7 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ef47adfe2e20a..e1fc1ec0c3775 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -2120,15 +2120,9 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
MI.setDesc(get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
break;
- case AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term:
- MI.setDesc(get(AMDGPU::V_CMPX_EQ_U32_nosdst_e64));
- break;
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
break;
- case AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term:
- MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e64));
- break;
case AMDGPU::SI_SPILL_S32_TO_VGPR:
MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
@@ -3246,9 +3240,7 @@ bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
case AMDGPU::S_AND_B32_term:
case AMDGPU::S_AND_SAVEEXEC_B32_term:
case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
- case AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term:
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
- case AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term:
break;
case AMDGPU::SI_IF:
case AMDGPU::SI_ELSE:
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 385c0e38018bc..a5fd1eb659994 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -496,29 +496,23 @@ def S_AND_B32_term : WrapTerminatorInst<S_AND_B32>;
def S_AND_SAVEEXEC_B32_term : WrapTerminatorInst<S_AND_SAVEEXEC_B32>;
}
-class WrapTerminatorVALU<InstSI base_inst> : VPseudoInstSI<
+class WrapTerminatorVOPC<VOPC_Pseudo base_inst> : VPseudoInstSI<
base_inst.OutOperandList,
base_inst.InOperandList> {
let Uses = base_inst.Uses;
let Defs = base_inst.Defs;
- let isTerminator = 1;
- let isConvergent = base_inst.isConvergent;
- let isCompare = base_inst.isCompare;
- let hasSideEffects = base_inst.hasSideEffects;
- let UseNamedOperandTable = base_inst.UseNamedOperandTable;
let SchedRW = base_inst.SchedRW;
- let VOPC = base_inst.VOPC;
+ let isTerminator = 1;
+ let isConvergent = 1;
+ let isCompare = 1;
+ let VOPC = 1;
}
let SubtargetPredicate = HasNoSdstCMPX in {
def V_CMPX_EQ_U32_nosdst_e32_term
- : WrapTerminatorVALU<V_CMPX_EQ_U32_nosdst_e32>;
-def V_CMPX_EQ_U32_nosdst_e64_term
- : WrapTerminatorVALU<V_CMPX_EQ_U32_nosdst_e64>;
+ : WrapTerminatorVOPC<V_CMPX_EQ_U32_nosdst_e32>;
def V_CMPX_EQ_U64_nosdst_e32_term
- : WrapTerminatorVALU<V_CMPX_EQ_U64_nosdst_e32>;
-def V_CMPX_EQ_U64_nosdst_e64_term
- : WrapTerminatorVALU<V_CMPX_EQ_U64_nosdst_e64>;
+ : WrapTerminatorVOPC<V_CMPX_EQ_U64_nosdst_e32>;
}
def WAVE_BARRIER : SPseudoInstSI<(outs), (ins),
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index 7c8fa130f16e0..d142d2f95776f 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -296,15 +296,9 @@ bool SIOptimizeExecMasking::removeTerminatorBit(MachineInstr &MI) const {
case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
return true;
- case AMDGPU::V_CMPX_EQ_U32_nosdst_e64_term:
- MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U32_nosdst_e64));
- return true;
case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
return true;
- case AMDGPU::V_CMPX_EQ_U64_nosdst_e64_term:
- MI.setDesc(TII->get(AMDGPU::V_CMPX_EQ_U64_nosdst_e64));
- return true;
default:
return false;
}
>From 78c021fbb2e68e9918fa577a02f5f3ff24b9b47c Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Fri, 5 Jun 2026 12:14:29 +0100
Subject: [PATCH 14/14] Format
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e1fc1ec0c3775..f3372884c62c9 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7181,11 +7181,10 @@ static void emitLoadScalarOpsFromVGPRLoop(
.addReg(VScalarOp);
if (UseNewExecInstructions) {
- auto CmpxMI =
- BuildMI(LoopBB, LoopBB.end(), DL,
- TII.get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
- .addReg(CurReg)
- .addReg(VScalarOp);
+ auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
+ TII.get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
+ .addReg(CurReg)
+ .addReg(VScalarOp);
if (I == LoopBB.end())
I = CmpxMI.getInstr()->getIterator();
} else {
@@ -7254,11 +7253,10 @@ static void emitLoadScalarOpsFromVGPRLoop(
NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
if (UseNewExecInstructions) {
- auto CmpxMI =
- BuildMI(LoopBB, LoopBB.end(), DL,
- TII.get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
- .addReg(CurReg)
- .addReg(VScalarOp, VScalarOpUndef, SubReg);
+ auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
+ TII.get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
+ .addReg(CurReg)
+ .addReg(VScalarOp, VScalarOpUndef, SubReg);
if (I == LoopBB.end())
I = CmpxMI.getInstr()->getIterator();
} else {
More information about the llvm-commits
mailing list