[llvm] [AMDGPU] Waterfall loop codegen improvement in SIInstrInfo (PR #192415)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 02:43:07 PDT 2026
https://github.com/gretay-amd created https://github.com/llvm/llvm-project/pull/192415
When generating waterfall loops, use the instructions `v_cmpx_eq_*` and `s_andn2_wrexec_*` as recommended for recent architectures, instead of `v_cmp_eq_*` and `s_and saveexec`.
This PR only updates waterfall loop code generation in `SIInstrInfo.cpp`. Other places that generated waterfall loops can be handled separately.
- Add new lane mask constants for `s_andn2_wrexec` and `v_cmpx_eq_u32/u64`
- Fix test `mubuf-legalize-operands.mir` to track liveness needed for verifying phi nodes
- Update .ll and .mir tests to accept the new instruction sequences
>From 1ff946c71b9a2f78a78c10cbc96de46884f96101 Mon Sep 17 00:00:00 2001
From: Greta Yorsh <Greta.Yorsh at amd.com>
Date: Tue, 31 Mar 2026 15:09:50 +0100
Subject: [PATCH] [AMDGPU] Waterfall loop codegen improvement
Use the instructions [v_cmpx_eq_*] and [s_andn2_wrexec_*] recommended
for waterfall loops in recent architectures.
This commit only updates code generation in SIInstrInfo.cpp.
Other places that generated waterfall loops can be handled separately.
- Add new lane mask constants for s_andn2_wrexec and v_cmpx_eq_u32/u64
- Fix one .mir test to track liveness needed for verifying phi nodes
- Update .ll and .mir tests to accept the new instruction sequences
---
llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h | 9 +
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 6 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 145 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 1308 ++++++++--------
.../buffer-fat-pointer-atomicrmw-fmax.ll | 1385 ++++++++---------
.../buffer-fat-pointer-atomicrmw-fmin.ll | 1385 ++++++++---------
...e92561-restore-undef-scc-verifier-error.ll | 69 +-
...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll | 74 +-
....amdgcn.struct.buffer.load.format.v3f16.ll | 136 +-
...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll | 74 +-
...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll | 146 +-
...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll | 146 +-
...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll | 222 ++-
...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll | 222 ++-
...gcn.struct.ptr.buffer.load.format.v3f16.ll | 80 +-
.../AMDGPU/move-to-valu-vimage-vsample.ll | 84 +-
...uf-legalize-operands-non-ptr-intrinsics.ll | 556 ++++---
.../CodeGen/AMDGPU/mubuf-legalize-operands.ll | 556 ++++---
.../AMDGPU/mubuf-legalize-operands.mir | 441 ++++--
...r-descriptor-waterfall-loop-idom-update.ll | 53 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 98 +-
llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll | 28 +-
22 files changed, 3658 insertions(+), 3565 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
index 3fdd962dcc30c..cac303dd22599 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULaneMaskUtils.h
@@ -27,10 +27,13 @@ class LaneMaskConstants {
const unsigned AndN2Opc;
const unsigned AndN2SaveExecOpc;
const unsigned AndN2TermOpc;
+ const unsigned AndN2WRExecOpc;
const unsigned AndSaveExecOpc;
const unsigned AndSaveExecTermOpc;
const unsigned BfmOpc;
const unsigned CMovOpc;
+ const unsigned CmpXEqU32Opc;
+ const unsigned CmpXEqU64Opc;
const unsigned CSelectOpc;
const unsigned MovOpc;
const unsigned MovTermOpc;
@@ -52,12 +55,18 @@ class LaneMaskConstants {
: AMDGPU::S_ANDN2_SAVEEXEC_B64),
AndN2TermOpc(IsWave32 ? AMDGPU::S_ANDN2_B32_term
: AMDGPU::S_ANDN2_B64_term),
+ AndN2WRExecOpc(IsWave32 ? AMDGPU::S_ANDN2_WREXEC_B32
+ : AMDGPU::S_ANDN2_WREXEC_B64),
AndSaveExecOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64),
AndSaveExecTermOpc(IsWave32 ? AMDGPU::S_AND_SAVEEXEC_B32_term
: AMDGPU::S_AND_SAVEEXEC_B64_term),
BfmOpc(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64),
CMovOpc(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
+ CmpXEqU32Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U32_nosdst_e32
+ : AMDGPU::V_CMPX_EQ_U32_nosdst_e64),
+ CmpXEqU64Opc(IsWave32 ? AMDGPU::V_CMPX_EQ_U64_nosdst_e32
+ : AMDGPU::V_CMPX_EQ_U64_nosdst_e64),
CSelectOpc(IsWave32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64),
MovOpc(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
MovTermOpc(IsWave32 ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term),
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 5994eeb54095b..5cd6c847d0bbd 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -182,6 +182,12 @@ class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
+
+ // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
+ // fail.
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties().setNoPHIs();
+ }
};
} // end anonymous namespace
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index bc3052b139d18..1128566cb36ce 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7092,22 +7092,43 @@ void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
// Emit the actual waterfall loop, executing the wrapped instruction for each
// unique value of \p ScalarOps across all lanes. In the best case we execute 1
// iteration, in the worst case we execute 64 (once per lane).
-static void
-emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
- MachineRegisterInfo &MRI,
- MachineBasicBlock &LoopBB,
- MachineBasicBlock &BodyBB,
- const DebugLoc &DL,
- ArrayRef<MachineOperand *> ScalarOps) {
+static void emitLoadScalarOpsFromVGPRLoop(
+ const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
+ MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
+ ArrayRef<MachineOperand *> ScalarOps) {
MachineFunction &MF = *LoopBB.getParent();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
+ // Emit [v_cmpx_eq] and [s_andn2_wrexec] when these instructions are
+ // available.
+ // Otherwise, use the previous pattern of [v_cmp_eq], [s_and_saveexec],
+ // and [s_xor].
+ // TODO: Accurately detect the availability of [s_andn2_wrexec] instruction
+ // in the target. For now, use the same condition as for the detection
+ // [v_cmpx_eq].
+ bool UseNewExecInstructions = ST.hasNoSdstCMPX();
+
MachineBasicBlock::iterator I = LoopBB.begin();
Register CondReg;
+ Register PhiExec = MRI.createVirtualRegister(BoolXExecRC);
+ Register NewExec = MRI.createVirtualRegister(BoolXExecRC);
+
+ if (UseNewExecInstructions) {
+ Register InitExec = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
+ .addReg(LMC.ExecReg);
+
+ BuildMI(LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
+ .addReg(InitExec)
+ .addMBB(&PredBB)
+ .addReg(NewExec)
+ .addMBB(&BodyBB);
+ }
+
for (MachineOperand *ScalarOp : ScalarOps) {
unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
unsigned NumSubRegs = RegSize / 32;
@@ -7119,21 +7140,27 @@ emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
.addReg(VScalarOp);
- Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+ if (UseNewExecInstructions) {
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU32Opc))
+ .addReg(CurReg)
+ .addReg(VScalarOp);
+ } else {
+ Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
- .addReg(CurReg)
- .addReg(VScalarOp);
+ BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
+ .addReg(CurReg)
+ .addReg(VScalarOp);
- // Combine the comparison results with AND.
- if (!CondReg) // First.
- CondReg = NewCondReg;
- else { // If not the first, we create an AND.
- Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
- .addReg(CondReg)
- .addReg(NewCondReg);
- CondReg = AndReg;
+ // Combine the comparison results with AND.
+ if (!CondReg) // First.
+ CondReg = NewCondReg;
+ else { // If not the first, we create an AND.
+ Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ .addReg(CondReg)
+ .addReg(NewCondReg);
+ CondReg = AndReg;
+ }
}
// Update ScalarOp operand to use the SGPR ScalarOp.
@@ -7171,25 +7198,36 @@ emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
.addReg(CurRegHi)
.addImm(AMDGPU::sub1);
- Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
- auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
- NewCondReg)
- .addReg(CurReg);
- if (NumSubRegs <= 2)
- Cmp.addReg(VScalarOp);
- else
- Cmp.addReg(VScalarOp, VScalarOpUndef,
- TRI->getSubRegFromChannel(Idx, 2));
-
- // Combine the comparison results with AND.
- if (!CondReg) // First.
- CondReg = NewCondReg;
- else { // If not the first, we create an AND.
- Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
- .addReg(CondReg)
- .addReg(NewCondReg);
- CondReg = AndReg;
+ if (UseNewExecInstructions) {
+ auto CmpX =
+ BuildMI(LoopBB, I, DL, TII.get(LMC.CmpXEqU64Opc)).addReg(CurReg);
+
+ if (NumSubRegs <= 2)
+ CmpX.addReg(VScalarOp);
+ else
+ CmpX.addReg(VScalarOp, VScalarOpUndef,
+ TRI->getSubRegFromChannel(Idx, 2));
+ } else {
+ Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+ auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
+ NewCondReg)
+ .addReg(CurReg);
+ if (NumSubRegs <= 2)
+ Cmp.addReg(VScalarOp);
+ else
+ Cmp.addReg(VScalarOp, VScalarOpUndef,
+ TRI->getSubRegFromChannel(Idx, 2));
+
+ // Combine the comparison results with AND.
+ if (!CondReg) // First.
+ CondReg = NewCondReg;
+ else { // If not the first, we create an AND.
+ Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
+ .addReg(CondReg)
+ .addReg(NewCondReg);
+ CondReg = AndReg;
+ }
}
} // End for loop.
@@ -7211,20 +7249,29 @@ emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII,
}
}
- Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
- MRI.setSimpleHint(SaveExec, CondReg);
+ Register SaveExec;
+ if (!UseNewExecInstructions) {
+ SaveExec = MRI.createVirtualRegister(BoolXExecRC);
+ MRI.setSimpleHint(SaveExec, CondReg);
- // Update EXEC to matching lanes, saving original to SaveExec.
- BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
- .addReg(CondReg, RegState::Kill);
+ // Update EXEC to matching lanes, saving original to SaveExec.
+ BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
+ .addReg(CondReg, RegState::Kill);
+ }
// The original instruction is here; we insert the terminators after it.
I = BodyBB.end();
- // Update EXEC, switch all done bits to 0 and all todo bits to 1.
- BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
- .addReg(LMC.ExecReg)
- .addReg(SaveExec);
+ if (UseNewExecInstructions) {
+ MRI.setSimpleHint(NewExec, PhiExec);
+ BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2WRExecOpc), NewExec)
+ .addReg(PhiExec);
+ } else {
+ // Update EXEC, switch all done bits to 0 and all todo bits to 1.
+ BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
+ .addReg(LMC.ExecReg)
+ .addReg(SaveExec);
+ }
BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
}
@@ -7322,7 +7369,7 @@ loadScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
}
}
- emitLoadScalarOpsFromVGPRLoop(TII, MRI, *LoopBB, *BodyBB, DL, ScalarOps);
+ emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps);
MachineBasicBlock::iterator First = RemainderBB->begin();
// Restore SCC
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index b5844049fd287..f27ef068911c8 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -372,31 +372,32 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
@@ -431,27 +432,25 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -461,60 +460,60 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB2_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f32_e32 v7, v8, v5
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB2_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB2_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -2310,66 +2309,66 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB10_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_add_f64_e32 v[11:12], v[13:14], v[5:6]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB10_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB10_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2409,65 +2408,61 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB10_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB10_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB10_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -2477,63 +2472,63 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
; GFX10-NEXT: v_mov_b32_e32 v7, v2
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v9
-; GFX10-NEXT: v_readfirstlane_b32 s9, v10
-; GFX10-NEXT: v_readfirstlane_b32 s10, v7
-; GFX10-NEXT: v_readfirstlane_b32 s11, v8
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[9:10]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[7:8]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:2048
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v7
+; GFX10-NEXT: v_readfirstlane_b32 s7, v8
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
+; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[4:7], 0 offen offset:2048
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB10_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB10_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v0, v11
; GFX10-NEXT: v_mov_b32_e32 v1, v12
; GFX10-NEXT: v_mov_b32_e32 v2, v13
; GFX10-NEXT: v_mov_b32_e32 v3, v14
; GFX10-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v9
-; GFX10-NEXT: v_readfirstlane_b32 s9, v10
-; GFX10-NEXT: v_readfirstlane_b32 s10, v7
-; GFX10-NEXT: v_readfirstlane_b32 s11, v8
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[9:10]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[7:8]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v9
+; GFX10-NEXT: v_readfirstlane_b32 s5, v10
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[9:10]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v7
+; GFX10-NEXT: v_readfirstlane_b32 s7, v8
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[7:8]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB10_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX10-NEXT: v_mov_b32_e32 v14, v1
; GFX10-NEXT: v_mov_b32_e32 v13, v0
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB10_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -4193,7 +4188,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4202,67 +4199,65 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4274,7 +4269,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4283,68 +4280,65 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v11, v7
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_add_f16_e32 v6, v6, v5
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4420,8 +4414,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4430,21 +4425,18 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
@@ -4452,43 +4444,41 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4496,8 +4486,9 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
@@ -4506,29 +4497,27 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v11, v7
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f16_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
@@ -4539,33 +4528,30 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4573,35 +4559,36 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v4, 3, v6
; GFX10-NEXT: v_and_b32_e32 v10, -4, v6
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
; GFX10-NEXT: v_not_b32_e32 v11, v7
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_add_f16_e32 v6, v6, v5
; GFX10-NEXT: v_lshlrev_b32_sdwa v6, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v6, v7, v11, v6
@@ -4609,32 +4596,31 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v4, v8
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -6015,7 +6001,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6024,80 +6012,77 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6109,7 +6094,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6118,79 +6105,76 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6273,8 +6257,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6283,21 +6268,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
@@ -6306,8 +6288,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
@@ -6328,34 +6311,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6363,8 +6343,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -6373,21 +6354,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
@@ -6396,8 +6374,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v10
@@ -6417,34 +6396,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6452,36 +6428,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_add_f32_e32 v4, v4, v10
; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
@@ -6494,32 +6471,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -7346,31 +7322,32 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -7405,124 +7382,120 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b32 v8, v4, s[4:7], 0 offen offset:1024
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b32 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB21_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_add_f16 v7, v8, v5
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mov_b32_e32 v6, v7
; GFX11-NEXT: v_mov_b32_e32 v7, v8
; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX11-NEXT: v_mov_b32_e32 v8, v6
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: v_mov_b32_e32 v0, v6
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v8, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_add_f16 v7, v8, v5
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
; GFX10-NEXT: v_mov_b32_e32 v8, v6
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -9669,31 +9642,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9776,24 +9750,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -9804,8 +9776,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, v6, v8 :: v_dual_add_f32 v5, v5, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -9826,58 +9799,53 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -9888,8 +9856,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v8
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -9910,57 +9879,54 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-FAKE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB28_3: ; %atomicrmw.start
@@ -9969,8 +9935,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_add_f32_e32 v5, v5, v8
; GFX10-NEXT: v_add_f32_e32 v6, v6, v9
; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
@@ -9988,32 +9955,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB28_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 3d85bf7019426..621d53c81345b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -363,31 +363,32 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_max_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -453,27 +454,25 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_max_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -483,26 +482,26 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_fmax v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: buffer_gl1_inv
@@ -1540,68 +1539,68 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_max_num_f64_e32 v[5:6], v[5:6], v[5:6]
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[11:12], v[0:1], v[5:6]
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB7_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB7_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -1641,25 +1640,23 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
@@ -1667,66 +1664,64 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[11:12], v[0:1], v[5:6]
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB7_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: v_mov_b32_e32 v1, v6
@@ -3312,7 +3307,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3321,70 +3318,67 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v5.l, v5.l
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3396,7 +3390,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3405,71 +3401,69 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v10, v5, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3547,8 +3541,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3557,21 +3552,18 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3580,45 +3572,42 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX11-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3626,8 +3615,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3636,21 +3626,18 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3658,8 +3645,9 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v10
@@ -3672,33 +3660,30 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3706,36 +3691,37 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
; GFX10-NEXT: v_max_f16_e32 v4, v4, v10
; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
@@ -3744,32 +3730,31 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -5160,7 +5145,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5169,80 +5156,77 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5254,7 +5238,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5263,79 +5249,76 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v4, v10
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5418,8 +5401,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5428,21 +5412,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
@@ -5451,8 +5432,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v10
@@ -5473,34 +5455,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5508,8 +5487,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5518,21 +5498,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
@@ -5541,8 +5518,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v4, v10
@@ -5562,34 +5540,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5597,36 +5572,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f32_e32 v4, v4, v10
; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
@@ -5639,32 +5615,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -6642,67 +6617,67 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v5, v7, v7
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: v_pk_max_num_f16 v6, v5, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mov_b32_e32 v5, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v7
; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-NEXT: v_mov_b32_e32 v7, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -6769,24 +6744,22 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
@@ -6794,103 +6767,101 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_pk_max_f16 v6, v5, v8
; GFX11-NEXT: v_mov_b32_e32 v5, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v7
; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-NEXT: v_mov_b32_e32 v7, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_pk_max_f16 v6, v5, v8
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -8266,40 +8237,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v6, v6, v8 :: v_dual_max_num_f32 v5, v5, v9
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8318,33 +8290,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8355,40 +8325,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v6, v6, v9 :: v_dual_max_num_f32 v5, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8407,33 +8378,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8517,24 +8486,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8545,8 +8512,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_max_f32 v6, v6, v8 :: v_dual_max_f32 v5, v5, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8567,58 +8535,53 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8629,8 +8592,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_max_f32 v6, v6, v9 :: v_dual_max_f32 v5, v5, v8
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8651,57 +8615,54 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
@@ -8710,8 +8671,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f32_e32 v5, v5, v8
; GFX10-NEXT: v_max_f32_e32 v6, v6, v9
; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
@@ -8729,32 +8691,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 3f15d35320573..a8b5c5a201068 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -363,31 +363,32 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_min_num_f32 v5, v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-NEXT: ; implicit-def: $vgpr4
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB2_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -453,27 +454,25 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX11-NEXT: buffer_atomic_min_f32 v5, v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-NEXT: ; implicit-def: $vgpr4
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: buffer_gl1_inv
@@ -483,26 +482,26 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[8:11], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_fmin v5, v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB2_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: buffer_gl1_inv
@@ -1540,68 +1539,68 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:2048
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], null offen offset:2048
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_max_num_f64_e32 v[5:6], v[5:6], v[5:6]
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[0:1], v[5:6]
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v9
-; GFX12-NEXT: v_readfirstlane_b32 s5, v10
-; GFX12-NEXT: v_readfirstlane_b32 s6, v7
-; GFX12-NEXT: v_readfirstlane_b32 s7, v8
+; GFX12-NEXT: v_readfirstlane_b32 s0, v9
+; GFX12-NEXT: v_readfirstlane_b32 s1, v10
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v7
+; GFX12-NEXT: v_readfirstlane_b32 s3, v8
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB7_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB7_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -1641,25 +1640,23 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:2048
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
+; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[0:3], 0 offen offset:2048
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
@@ -1667,66 +1664,64 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_min_f64 v[11:12], v[0:1], v[5:6]
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v9
-; GFX11-NEXT: v_readfirstlane_b32 s5, v10
-; GFX11-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-NEXT: v_readfirstlane_b32 s7, v8
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v9
+; GFX11-NEXT: v_readfirstlane_b32 s1, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v7
+; GFX11-NEXT: v_readfirstlane_b32 s3, v8
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[7:8]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB7_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB7_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]
; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB7_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[8:11], 0 offen offset:2048 glc
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[5:6], v4, s[4:7], 0 offen offset:2048 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: v_mov_b32_e32 v1, v6
@@ -3312,7 +3307,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3321,70 +3318,67 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v5.l, v5.l
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v4.h, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3396,7 +3390,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3405,71 +3401,69 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v10, v5, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v4, v4, v10
; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3547,8 +3541,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
@@ -3557,21 +3552,18 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3580,45 +3572,42 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v4.h, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX11-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3626,8 +3615,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -3636,21 +3626,18 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
@@ -3658,8 +3645,9 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
; GFX11-FAKE16-NEXT: v_min_f16_e32 v4, v4, v10
@@ -3672,33 +3660,30 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3706,36 +3691,37 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
; GFX10-NEXT: v_min_f16_e32 v4, v4, v10
; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
@@ -3744,32 +3730,31 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -5160,7 +5145,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5169,80 +5156,77 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5254,7 +5238,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5263,79 +5249,76 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v4, v10
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5418,8 +5401,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5428,21 +5412,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
@@ -5451,8 +5432,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v10
@@ -5473,34 +5455,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5508,8 +5487,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
@@ -5518,21 +5498,18 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
@@ -5541,8 +5518,9 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v4, v10
@@ -5562,34 +5540,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5597,36 +5572,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB15_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_min_f32_e32 v4, v4, v10
; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v4
@@ -5639,32 +5615,31 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB15_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB15_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -6642,67 +6617,67 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s5, s4
; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-NEXT: s_mov_b32 exec_lo, s4
; GFX12-NEXT: v_pk_max_num_f16 v8, v5, v5
-; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_pk_max_num_f16 v5, v7, v7
-; GFX12-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: v_pk_min_num_f16 v6, v5, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mov_b32_e32 v5, v6
; GFX12-NEXT: v_mov_b32_e32 v6, v7
; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 exec_lo, s5
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-NEXT: v_mov_b32_e32 v7, v5
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-NEXT: v_mov_b32_e32 v0, v5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -6769,24 +6744,22 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s1, 0
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
@@ -6794,103 +6767,101 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_pk_min_f16 v6, v5, v8
; GFX11-NEXT: v_mov_b32_e32 v5, v6
; GFX11-NEXT: v_mov_b32_e32 v6, v7
; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-NEXT: v_mov_b32_e32 v7, v5
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB18_3
; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: v_mov_b32_e32 v0, v5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_pk_min_f16 v6, v5, v8
; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB18_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -8266,40 +8237,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v6, v6, v8 :: v_dual_min_num_f32 v5, v5, v9
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8318,33 +8290,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8355,40 +8325,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], null offen offset:1024
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB21_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v6, v6, v9 :: v_dual_min_num_f32 v5, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8407,33 +8378,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8517,24 +8486,22 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8545,8 +8512,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_min_f32 v6, v6, v8 :: v_dual_min_f32 v5, v5, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8567,58 +8535,53 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -8629,8 +8592,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_dual_min_f32 v6, v6, v9 :: v_dual_min_f32 v5, v5, v8
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
@@ -8651,57 +8615,54 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-FAKE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, 0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
-; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX10-NEXT: buffer_load_dword v7, v4, s[4:7], 0 offen offset:1024
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start
@@ -8710,8 +8671,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: s_mov_b32 s10, s9
; GFX10-NEXT: v_min_f32_e32 v5, v5, v8
; GFX10-NEXT: v_min_f32_e32 v6, v6, v9
; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
@@ -8729,32 +8691,31 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_mov_b32_e32 v6, v7
; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT: v_readfirstlane_b32 s8, v0
-; GFX10-NEXT: v_readfirstlane_b32 s9, v1
-; GFX10-NEXT: v_readfirstlane_b32 s10, v2
-; GFX10-NEXT: v_readfirstlane_b32 s11, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=1
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: s_cbranch_execnz .LBB21_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
index 9644c941cd06c..db6b962b8a443 100644
--- a/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue92561-restore-undef-scc-verifier-error.ll
@@ -15,50 +15,47 @@ define void @issue92561(ptr addrspace(1) %arg) {
; SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off offset:16
; SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off
; SDAG-NEXT: v_mov_b32_e32 v8, 0
-; SDAG-NEXT: s_mov_b32 s12, 0
-; SDAG-NEXT: s_mov_b32 s3, exec_lo
+; SDAG-NEXT: s_mov_b32 s8, 0
+; SDAG-NEXT: s_mov_b32 s12, exec_lo
+; SDAG-NEXT: s_mov_b32 s9, s8
+; SDAG-NEXT: s_mov_b32 s10, s8
+; SDAG-NEXT: s_mov_b32 s11, s8
; SDAG-NEXT: s_mov_b32 s13, s12
-; SDAG-NEXT: s_mov_b32 s14, s12
-; SDAG-NEXT: s_mov_b32 s15, s12
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; SDAG-NEXT: v_readfirstlane_b32 s6, v2
-; SDAG-NEXT: v_readfirstlane_b32 s7, v3
-; SDAG-NEXT: v_readfirstlane_b32 s8, v4
-; SDAG-NEXT: v_readfirstlane_b32 s9, v5
-; SDAG-NEXT: v_readfirstlane_b32 s10, v6
-; SDAG-NEXT: v_readfirstlane_b32 s11, v7
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; SDAG-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; SDAG-NEXT: v_cmp_eq_u64_e64 s1, s[8:9], v[4:5]
-; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; SDAG-NEXT: v_cmp_eq_u64_e64 s2, s[10:11], v[6:7]
-; SDAG-NEXT: s_and_b32 s0, vcc_lo, s0
-; SDAG-NEXT: s_and_b32 s0, s0, s1
-; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; SDAG-NEXT: s_and_b32 s0, s0, s2
-; SDAG-NEXT: s_and_saveexec_b32 s0, s0
-; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[4:11], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; SDAG-NEXT: v_readfirstlane_b32 s2, v2
+; SDAG-NEXT: v_readfirstlane_b32 s3, v3
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; SDAG-NEXT: v_readfirstlane_b32 s4, v4
+; SDAG-NEXT: v_readfirstlane_b32 s5, v5
+; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; SDAG-NEXT: v_readfirstlane_b32 s6, v6
+; SDAG-NEXT: v_readfirstlane_b32 s7, v7
+; SDAG-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; SDAG-NEXT: image_sample_c_lz v9, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: s_and_not1_wrexec_b32 s13, s13
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; SDAG-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; SDAG-NEXT: s_cbranch_execnz .LBB0_1
; SDAG-NEXT: ; %bb.2:
-; SDAG-NEXT: s_mov_b32 exec_lo, s3
+; SDAG-NEXT: s_mov_b32 exec_lo, s12
; SDAG-NEXT: v_dual_mov_b32 v0, 0x7fc00000 :: v_dual_mov_b32 v1, 1.0
-; SDAG-NEXT: s_mov_b32 s0, s12
-; SDAG-NEXT: s_mov_b32 s1, s12
-; SDAG-NEXT: s_mov_b32 s2, s12
-; SDAG-NEXT: s_mov_b32 s3, s12
-; SDAG-NEXT: s_mov_b32 s4, s12
-; SDAG-NEXT: s_mov_b32 s5, s12
-; SDAG-NEXT: s_mov_b32 s6, s12
-; SDAG-NEXT: s_mov_b32 s7, s12
+; SDAG-NEXT: s_mov_b32 s0, s8
+; SDAG-NEXT: s_mov_b32 s1, s8
+; SDAG-NEXT: s_mov_b32 s2, s8
+; SDAG-NEXT: s_mov_b32 s3, s8
+; SDAG-NEXT: s_mov_b32 s4, s8
+; SDAG-NEXT: s_mov_b32 s5, s8
+; SDAG-NEXT: s_mov_b32 s6, s8
+; SDAG-NEXT: s_mov_b32 s7, s8
; SDAG-NEXT: s_clause 0x2
-; SDAG-NEXT: image_sample_c_lz v0, [v8, v8, v0, v8], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
-; SDAG-NEXT: image_sample_c_lz v1, [v8, v1, v8, v8], s[0:7], s[12:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v0, [v8, v8, v0, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v2, [v8, v8, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; SDAG-NEXT: image_sample_c_lz v1, [v8, v1, v8, v8], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
; SDAG-NEXT: s_waitcnt vmcnt(2)
; SDAG-NEXT: v_dual_add_f32 v0, v9, v0 :: v_dual_mov_b32 v9, v8
; SDAG-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index 874b336a929a5..a08929ee34529 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -104,33 +104,34 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v6
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v6
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[4:7], s3 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[0:3], s6 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr6
; GFX1200-NEXT: ; implicit-def: $vgpr5
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
@@ -140,33 +141,30 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_dual_mov_b32 v11, v4 :: v_dual_mov_b32 v10, v3
; GFX1250-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v8, v1
; GFX1250-NEXT: v_add_nc_u32_e32 v1, 0x80, v5
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v8
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v9
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v10
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v11
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v6
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[8:9]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[10:11]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v6
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s5, s4
+; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[8:9]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v10
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v11
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[10:11]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v6
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[4:7], s3 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s6 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11
; GFX1250-NEXT: ; implicit-def: $vgpr6
; GFX1250-NEXT: ; implicit-def: $vgpr1
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 128
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 24fcb933bf801..0395a94a057d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -12,24 +12,24 @@
define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
; GFX10-LABEL: main:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-NEXT: s_mov_b32 s5, s4
; GFX10-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: v_readfirstlane_b32 s6, v2
-; GFX10-NEXT: v_readfirstlane_b32 s7, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX10-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX10-NEXT: s_and_saveexec_b32 s0, s0
-; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s1
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v6
@@ -92,25 +92,24 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX11-TRUE16-LABEL: main:
; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
@@ -119,25 +118,24 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX11-FAKE16-LABEL: main:
; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
@@ -145,27 +143,27 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX12-TRUE16-LABEL: main:
; GFX12-TRUE16: ; %bb.0: ; %bb
-; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], null idxen
+; GFX12-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-TRUE16-NEXT: ; %bb.2:
-; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
@@ -174,27 +172,27 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
;
; GFX12-FAKE16-LABEL: main:
; GFX12-FAKE16: ; %bb.0: ; %bb
-; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], null idxen
+; GFX12-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], null idxen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX12-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-FAKE16-NEXT: ; %bb.2:
-; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 6921cca5a2394..20c70359c8558 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -40,33 +40,34 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB2_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
%ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret <2 x bfloat> %ret
@@ -80,32 +81,33 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[4:7], s3 idxen offen
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB3_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
%ret = call <2 x bfloat> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index d6e140c14d4cc..8455266c6cd15 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -296,32 +296,33 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[4:7], s3 idxen offen
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -331,33 +332,31 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -470,32 +469,33 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[4:7], s3 idxen offen
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr0
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -505,33 +505,31 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
+; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr0
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index d99f1a4a5b996..3fba31a46125d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -245,33 +245,34 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB4_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -281,33 +282,31 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB4_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -391,33 +390,34 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB5_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
@@ -427,33 +427,31 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s5, s4
; GFX1250-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB5_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index 3a6cea74de9e9..ec925fa40721f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -461,51 +461,50 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX10-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s2, s1
; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s6, v3
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX11-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s1, s1
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -516,28 +515,29 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s1, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s2, s1
; GFX1200-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX1200-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s1
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
@@ -547,28 +547,27 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s1, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[4:5]
-; GFX1250-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
@@ -637,60 +636,57 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX10-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_readfirstlane_b32 s7, v7
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, s7, v7
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_b32 s4, s4, s5
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
+; GFX10-NEXT: v_readfirstlane_b32 s10, v7
+; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[8:11], s7 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmax v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: v_readfirstlane_b32 s3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_and_b32 s0, s0, s1
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v3
+; GFX11-NEXT: v_readfirstlane_b32 s3, v4
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 glc
+; GFX11-NEXT: buffer_atomic_max_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -701,33 +697,34 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_max_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
@@ -737,33 +734,30 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s5, s4
+; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_max_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index a9f843f4a184a..9ee14d4c59915 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -461,51 +461,50 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX10-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s16 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s16 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s5
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
+; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s2, s1
; GFX11-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_readfirstlane_b32 s4, v1
; GFX11-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX11-NEXT: v_readfirstlane_b32 s6, v3
; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX11-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s1, s1
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -516,28 +515,29 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s1, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s2, s1
; GFX1200-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX1200-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1200-NEXT: s_cbranch_execnz .LBB8_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s1
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
@@ -547,28 +547,27 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: s_mov_b32 s1, exec_lo
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s1, s[6:7], v[4:5]
-; GFX1250-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s1, s1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s2, s2
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s1
; GFX1250-NEXT: s_cbranch_execnz .LBB8_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s1
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
@@ -637,60 +636,57 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX10-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_mov_b32 s8, exec_lo
+; GFX10-NEXT: s_mov_b32 s9, s8
; GFX10-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s8, v1
-; GFX10-NEXT: v_readfirstlane_b32 s9, v2
-; GFX10-NEXT: v_readfirstlane_b32 s10, v3
-; GFX10-NEXT: v_readfirstlane_b32 s11, v4
-; GFX10-NEXT: v_readfirstlane_b32 s7, v7
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[1:2]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[3:4]
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, s7, v7
-; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX10-NEXT: s_and_b32 s4, s4, s5
-; GFX10-NEXT: s_and_saveexec_b32 s4, s4
+; GFX10-NEXT: v_readfirstlane_b32 s4, v1
+; GFX10-NEXT: v_readfirstlane_b32 s5, v2
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[1:2]
+; GFX10-NEXT: v_readfirstlane_b32 s6, v3
+; GFX10-NEXT: v_readfirstlane_b32 s7, v4
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[3:4]
+; GFX10-NEXT: v_readfirstlane_b32 s10, v7
+; GFX10-NEXT: v_cmpx_eq_u32_e32 s10, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[8:11], s7 idxen offen offset:256 glc
+; GFX10-NEXT: buffer_atomic_fmin v0, v[5:6], s[4:7], s10 idxen offen offset:256 glc
+; GFX10-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX10-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX10-NEXT: ; implicit-def: $vgpr7
; GFX10-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s6
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s8
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s2, exec_lo
-; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-NEXT: v_readfirstlane_b32 s5, v2
-; GFX11-NEXT: v_readfirstlane_b32 s6, v3
-; GFX11-NEXT: v_readfirstlane_b32 s7, v4
-; GFX11-NEXT: v_readfirstlane_b32 s3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_and_b32 s0, s0, s1
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
+; GFX11-NEXT: s_mov_b32 s5, s4
+; GFX11-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v3
+; GFX11-NEXT: v_readfirstlane_b32 s3, v4
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX11-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 glc
+; GFX11-NEXT: buffer_atomic_min_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 glc
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX11-NEXT: ; implicit-def: $vgpr7
; GFX11-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -701,33 +697,34 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: s_mov_b32 s2, exec_lo
+; GFX1200-NEXT: s_mov_b32 s4, exec_lo
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_mov_b32 s5, s4
; GFX1200-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1200-NEXT: v_readfirstlane_b32 s4, v1
-; GFX1200-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-NEXT: v_readfirstlane_b32 s7, v4
-; GFX1200-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT: v_readfirstlane_b32 s0, v1
+; GFX1200-NEXT: v_readfirstlane_b32 s1, v2
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, s1
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[1:2]
+; GFX1200-NEXT: v_readfirstlane_b32 s2, v3
+; GFX1200-NEXT: v_readfirstlane_b32 s3, v4
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[3:4]
+; GFX1200-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1200-NEXT: s_wait_loadcnt 0x0
-; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: buffer_atomic_min_num_f32 v0, v[5:6], s[0:3], s6 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1200-NEXT: ; implicit-def: $vgpr7
; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1200-NEXT: s_cbranch_execnz .LBB9_1
; GFX1200-NEXT: ; %bb.2:
-; GFX1200-NEXT: s_mov_b32 exec_lo, s2
+; GFX1200-NEXT: s_mov_b32 exec_lo, s4
; GFX1200-NEXT: s_wait_loadcnt 0x0
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
@@ -737,33 +734,30 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
; GFX1250-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
; GFX1250-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
-; GFX1250-NEXT: s_mov_b32 s2, exec_lo
-; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-NEXT: v_readfirstlane_b32 s5, v3
-; GFX1250-NEXT: v_readfirstlane_b32 s6, v4
-; GFX1250-NEXT: v_readfirstlane_b32 s7, v5
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7
-; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1250-NEXT: s_and_b32 s0, s0, s1
+; GFX1250-NEXT: s_mov_b32 s4, exec_lo
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s5, s4
+; GFX1250-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX1250-NEXT: v_readfirstlane_b32 s2, v4
+; GFX1250-NEXT: v_readfirstlane_b32 s3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX1250-NEXT: v_readfirstlane_b32 s6, v7
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 s6, v7
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: buffer_atomic_min_num_f32 v0, v[8:9], s[0:3], s6 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: ; implicit-def: $vgpr7
; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_cbranch_execnz .LBB9_1
; GFX1250-NEXT: ; %bb.2:
-; GFX1250-NEXT: s_mov_b32 exec_lo, s2
+; GFX1250-NEXT: s_mov_b32 exec_lo, s4
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%voffset.add = add i32 %voffset, 256
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
index f64f8a81a256f..41cc904135beb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.v3f16.ll
@@ -9,24 +9,24 @@
define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
; GFX10-LABEL: main:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: s_mov_b32 s4, exec_lo
+; GFX10-NEXT: s_mov_b32 s5, s4
; GFX10-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: v_readfirstlane_b32 s5, v1
-; GFX10-NEXT: v_readfirstlane_b32 s6, v2
-; GFX10-NEXT: v_readfirstlane_b32 s7, v3
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX10-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX10-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX10-NEXT: s_and_saveexec_b32 s0, s0
-; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX10-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-NEXT: v_readfirstlane_b32 s3, v3
+; GFX10-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX10-NEXT: buffer_load_format_d16_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX10-NEXT: s_andn2_wrexec_b32 s5, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX10-NEXT: ; implicit-def: $vgpr4
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
; GFX10-NEXT: ; %bb.2:
-; GFX10-NEXT: s_mov_b32 exec_lo, s1
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v6
@@ -89,25 +89,24 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
;
; GFX11-TRUE16-LABEL: main:
; GFX11-TRUE16: ; %bb.0: ; %bb
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX11-TRUE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2:
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
@@ -116,25 +115,24 @@ define amdgpu_gs void @main(ptr addrspace(8) %arg, i32 %arg1) {
;
; GFX11-FAKE16-LABEL: main:
; GFX11-FAKE16: ; %bb.0: ; %bb
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v1
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[4:7], 0 idxen
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX11-FAKE16-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_d16_format_xyz v[5:6], v4, s[0:3], 0 idxen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-FAKE16-NEXT: ; %bb.2:
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v1, 0xffff, v6
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
index 10d363648e3ae..e003e9d33fc2a 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-vimage-vsample.ll
@@ -19,37 +19,35 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3, [[COPY3]], %subreg.sub4, [[COPY2]], %subreg.sub5, [[COPY1]], %subreg.sub6, [[COPY]], %subreg.sub7
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.1:
; GFX11-NEXT: successors: %bb.2(0x80000000)
; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %14, %bb.2
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX11-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_LOAD_V1_V2_nsa_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_nsa_gfx11 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -73,37 +71,35 @@ define amdgpu_ps float @vimage_move_to_valu(<8 x i32> %rsrc) {
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_256 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3, [[COPY3]], %subreg.sub4, [[COPY2]], %subreg.sub5, [[COPY1]], %subreg.sub6, [[COPY]], %subreg.sub7
; GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.1:
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %14, %bb.2
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_LOAD_V1_V2_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_LOAD_V1_V2_gfx12 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE5]], 1, 1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -137,37 +133,35 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.1:
; GFX11-NEXT: successors: %bb.2(0x80000000)
; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX11-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -196,37 +190,35 @@ define amdgpu_ps float @vsample_move_to_valu_rsrc(<8 x i32> %rsrc, <4 x i32> inr
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.1:
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U64_e64_2]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_1]], [[V_CMP_EQ_U64_e64_3]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE5]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_256 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3, [[V_READFIRSTLANE_B32_4]], %subreg.sub4, [[V_READFIRSTLANE_B32_5]], %subreg.sub5, [[V_READFIRSTLANE_B32_6]], %subreg.sub6, [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE6]], [[REG_SEQUENCE1]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
@@ -260,27 +252,27 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.1:
; GFX11-NEXT: successors: %bb.2(0x80000000)
; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub0, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub1, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit $exec
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub2, implicit $exec
; GFX11-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub3, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX11-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit $exec
- ; GFX11-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX11-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX11-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.2:
; GFX11-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx11_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx11 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX11-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX11-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.3:
@@ -309,27 +301,27 @@ define amdgpu_ps float @vsample_move_to_valu_samp(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
; GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; GFX12-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.1:
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %19, %bb.2
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub0, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub1, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit $exec
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]].sub0_sub1, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub2, implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE1]].sub3, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit $exec
- ; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; GFX12-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], [[REG_SEQUENCE1]].sub2_sub3, implicit-def $exec, implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[IMAGE_SAMPLE_V1_V1_gfx12_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx12 [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], killed [[REG_SEQUENCE4]], 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; GFX12-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index 46d63c87656d9..48bd02e97eadf 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -38,24 +38,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W32-LABEL: mubuf_vgpr:
; GFX1010_W32: ; %bb.0:
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
@@ -63,24 +63,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1010_W64-LABEL: mubuf_vgpr:
; GFX1010_W64: ; %bb.0:
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
@@ -88,25 +88,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W32-LABEL: mubuf_vgpr:
; GFX1100_W32: ; %bb.0:
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
@@ -114,25 +113,24 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; GFX1100_W64-LABEL: mubuf_vgpr:
; GFX1100_W64: ; %bb.0:
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
@@ -293,41 +291,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W32: ; %bb.0: ; %entry
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W32-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -339,41 +337,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1010_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W64: ; %bb.0: ; %entry
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W64-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -385,44 +383,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W32: ; %bb.0: ; %entry
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
+; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W32-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -434,44 +429,41 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; GFX1100_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W64: ; %bb.0: ; %entry
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
+; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W64-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -770,53 +762,53 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s5, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s5, vcc_lo, s5
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, s5
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1010_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1010_W32-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1010_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W32-NEXT: ; %bb.3: ; %bb1
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s10, s9
; GFX1010_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s9
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -826,53 +818,53 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[6:7], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], s[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1010_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX1010_W64-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1010_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W64-NEXT: ; %bb.3: ; %bb1
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[10:11]
; GFX1010_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[10:11]
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -882,57 +874,55 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[10:11], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s5
+; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1100_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W32-NEXT: ; %bb.3: ; %bb1
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W32-NEXT: s_mov_b32 s2, exec_lo
-; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s4
+; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s2
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: .LBB2_6: ; %bb2
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -942,57 +932,55 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[10:11], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1100_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W64-NEXT: ; %bb.3: ; %bb1
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[8:9], exec
-; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: .LBB2_6: ; %bb2
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index b7947de7d5836..0f70397a9fb8e 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -37,24 +37,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W32-LABEL: mubuf_vgpr:
; GFX1010_W32: ; %bb.0:
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
@@ -62,24 +62,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1010_W64-LABEL: mubuf_vgpr:
; GFX1010_W64: ; %bb.0:
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
@@ -87,25 +87,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W32-LABEL: mubuf_vgpr:
; GFX1100_W32: ; %bb.0:
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
@@ -113,25 +112,24 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; GFX1100_W64-LABEL: mubuf_vgpr:
; GFX1100_W64: ; %bb.0:
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v5, v4, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB0_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, v5
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
@@ -299,41 +297,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W32: ; %bb.0: ; %entry
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
-; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
+; GFX1010_W32-NEXT: s_mov_b32 s8, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, s8
; GFX1010_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s9, s9
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W32-NEXT: ; %bb.4:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W32-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -345,41 +343,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1010_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1010_W64: ; %bb.0: ; %entry
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[0:1]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v2
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v3
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
-; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1010_W64-NEXT: s_mov_b64 s[8:9], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX1010_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[10:11], s[10:11]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1010_W64-NEXT: ; %bb.4:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1010_W64-NEXT: global_store_dword v[9:10], v13, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -391,44 +389,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W32-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W32: ; %bb.0: ; %entry
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
+; GFX1100_W32-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
+; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, s4
; GFX1100_W32-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W32-NEXT: ; %bb.4:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W32-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
@@ -440,44 +435,41 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; GFX1100_W64-LABEL: mubuf_vgpr_adjacent_in_block:
; GFX1100_W64: ; %bb.0: ; %entry
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX1100_W64-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v13, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
+; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX1100_W64-NEXT: .LBB1_3: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
-; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v0, v8, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[6:7], s[6:7]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB1_3
; GFX1100_W64-NEXT: ; %bb.4:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(1)
; GFX1100_W64-NEXT: global_store_b32 v[9:10], v13, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
@@ -787,53 +779,53 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W32-NEXT: ;;#ASMSTART
; GFX1010_W32-NEXT: s_mov_b32 s4, 17
; GFX1010_W32-NEXT: ;;#ASMEND
-; GFX1010_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W32-NEXT: s_mov_b32 s5, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s6, s5
; GFX1010_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[8:9], v[0:1]
; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s5, s[10:11], v[2:3]
-; GFX1010_W32-NEXT: s_and_b32 s5, vcc_lo, s5
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, s5
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[10:11], v[2:3]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s6, s6
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: ; implicit-def: $vgpr9
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W32-NEXT: ; %bb.2:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1010_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s5, vcc_lo
+; GFX1010_W32-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1010_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W32-NEXT: ; %bb.3: ; %bb1
; GFX1010_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W32-NEXT: s_mov_b32 s6, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s9, exec_lo
+; GFX1010_W32-NEXT: s_mov_b32 s10, s9
; GFX1010_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W32-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[4:5]
-; GFX1010_W32-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[6:7]
-; GFX1010_W32-NEXT: s_and_b32 s4, vcc_lo, s4
-; GFX1010_W32-NEXT: s_and_saveexec_b32 s4, s4
-; GFX1010_W32-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W32-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W32-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[6:7]
+; GFX1010_W32-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W32-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX1010_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GFX1010_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W32-NEXT: ; %bb.5:
-; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s6
+; GFX1010_W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W32-NEXT: s_mov_b32 exec_lo, s9
; GFX1010_W32-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1010_W32-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1010_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W32-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W32-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -843,53 +835,53 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1010_W64-NEXT: ;;#ASMSTART
; GFX1010_W64-NEXT: s_mov_b32 s4, 17
; GFX1010_W64-NEXT: ;;#ASMEND
-; GFX1010_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1010_W64-NEXT: s_mov_b64 s[6:7], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[6:7]
; GFX1010_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v0
; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v1
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[8:9], v[0:1]
; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v2
; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[6:7], s[10:11], v[2:3]
-; GFX1010_W64-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], s[6:7]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[10:11], v[2:3]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v9, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1010_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: ; implicit-def: $vgpr9
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1010_W64-NEXT: ; %bb.2:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1010_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX1010_W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX1010_W64-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1010_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1010_W64-NEXT: ; %bb.3: ; %bb1
; GFX1010_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1010_W64-NEXT: s_mov_b64 s[12:13], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[10:11], exec
+; GFX1010_W64-NEXT: s_mov_b64 s[12:13], s[10:11]
; GFX1010_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s8, v4
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s9, v5
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s10, v6
-; GFX1010_W64-NEXT: v_readfirstlane_b32 s11, v7
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[4:5]
-; GFX1010_W64-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[6:7]
-; GFX1010_W64-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
-; GFX1010_W64-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
-; GFX1010_W64-NEXT: buffer_load_format_x v9, v0, s[8:11], 0 idxen
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s4, v4
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s5, v5
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[4:5], v[4:5]
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s6, v6
+; GFX1010_W64-NEXT: v_readfirstlane_b32 s7, v7
+; GFX1010_W64-NEXT: v_cmpx_eq_u64_e64 s[6:7], v[6:7]
+; GFX1010_W64-NEXT: buffer_load_format_x v8, v0, s[4:7], 0 idxen
+; GFX1010_W64-NEXT: s_andn2_wrexec_b64 s[12:13], s[12:13]
; GFX1010_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1010_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1010_W64-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1010_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1010_W64-NEXT: ; %bb.5:
-; GFX1010_W64-NEXT: s_mov_b64 exec, s[12:13]
+; GFX1010_W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1010_W64-NEXT: s_mov_b64 exec, s[10:11]
; GFX1010_W64-NEXT: .LBB2_6: ; %bb2
-; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1010_W64-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1010_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1010_W64-NEXT: global_store_dword v[11:12], v9, off
+; GFX1010_W64-NEXT: global_store_dword v[11:12], v8, off
; GFX1010_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1010_W64-NEXT: s_setpc_b64 s[30:31]
;
@@ -899,57 +891,55 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W32-NEXT: ;;#ASMSTART
; GFX1100_W32-NEXT: s_mov_b32 s4, 17
; GFX1100_W32-NEXT: ;;#ASMEND
-; GFX1100_W32-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[10:11], v[2:3]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s5
+; GFX1100_W32-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W32-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1100_W32-NEXT: ; implicit-def: $vgpr9
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W32-NEXT: ; %bb.2:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s1
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s5
; GFX1100_W32-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX1100_W32-NEXT: s_mov_b32 s5, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W32-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W32-NEXT: ; %bb.3: ; %bb1
; GFX1100_W32-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W32-NEXT: s_mov_b32 s2, exec_lo
-; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W32-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; GFX1100_W32-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[6:7]
-; GFX1100_W32-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1100_W32-NEXT: s_mov_b32 s4, exec_lo
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_and_saveexec_b32 s0, s0
-; GFX1100_W32-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W32-NEXT: s_mov_b32 s6, s4
+; GFX1100_W32-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[4:5]
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W32-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W32-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[6:7]
+; GFX1100_W32-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W32-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX1100_W32-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W32-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1100_W32-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W32-NEXT: ; %bb.5:
-; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s2
+; GFX1100_W32-NEXT: s_mov_b32 exec_lo, s4
; GFX1100_W32-NEXT: .LBB2_6: ; %bb2
; GFX1100_W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1100_W32-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX1100_W32-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W32-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W32-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W32-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W32-NEXT: s_setpc_b64 s[30:31]
;
@@ -959,57 +949,55 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; GFX1100_W64-NEXT: ;;#ASMSTART
; GFX1100_W64-NEXT: s_mov_b32 s4, 17
; GFX1100_W64-NEXT: ;;#ASMEND
-; GFX1100_W64-NEXT: v_mov_b32_e32 v8, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
-; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s8, v0
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s9, v1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s10, v2
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s11, v3
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[10:11], v[2:3]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: v_mov_b32_e32 v9, s4
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v8, s[8:11], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX1100_W64-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[0:1]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[2:3]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v9, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX1100_W64-NEXT: ; implicit-def: $vgpr8
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1100_W64-NEXT: ; implicit-def: $vgpr9
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_1
; GFX1100_W64-NEXT: ; %bb.2:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[6:7]
; GFX1100_W64-NEXT: v_and_b32_e32 v0, 0x3ff, v31
-; GFX1100_W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX1100_W64-NEXT: s_mov_b64 s[6:7], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100_W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1100_W64-NEXT: s_cbranch_execz .LBB2_6
; GFX1100_W64-NEXT: ; %bb.3: ; %bb1
; GFX1100_W64-NEXT: v_mov_b32_e32 v0, s4
-; GFX1100_W64-NEXT: s_mov_b64 s[8:9], exec
-; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s4, v4
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s5, v5
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s6, v6
-; GFX1100_W64-NEXT: v_readfirstlane_b32 s7, v7
-; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[4:5]
-; GFX1100_W64-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[6:7]
-; GFX1100_W64-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; GFX1100_W64-NEXT: s_mov_b64 s[4:5], exec
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX1100_W64-NEXT: buffer_load_format_x v9, v0, s[4:7], 0 idxen
+; GFX1100_W64-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX1100_W64-NEXT: .LBB2_4: ; =>This Inner Loop Header: Depth=1
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1100_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[0:1], v[4:5]
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s2, v6
+; GFX1100_W64-NEXT: v_readfirstlane_b32 s3, v7
+; GFX1100_W64-NEXT: v_cmpx_eq_u64_e64 s[2:3], v[6:7]
+; GFX1100_W64-NEXT: buffer_load_format_x v8, v0, s[0:3], 0 idxen
+; GFX1100_W64-NEXT: s_and_not1_wrexec_b64 s[8:9], s[8:9]
; GFX1100_W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
; GFX1100_W64-NEXT: ; implicit-def: $vgpr0
-; GFX1100_W64-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX1100_W64-NEXT: s_cbranch_execnz .LBB2_4
; GFX1100_W64-NEXT: ; %bb.5:
-; GFX1100_W64-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1100_W64-NEXT: s_mov_b64 exec, s[4:5]
; GFX1100_W64-NEXT: .LBB2_6: ; %bb2
; GFX1100_W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1100_W64-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX1100_W64-NEXT: s_waitcnt vmcnt(0)
-; GFX1100_W64-NEXT: global_store_b32 v[11:12], v9, off dlc
+; GFX1100_W64-NEXT: global_store_b32 v[11:12], v8, off dlc
; GFX1100_W64-NEXT: s_waitcnt_vscnt null, 0x0
; GFX1100_W64-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 6ef1574c148b5..5429461bef033 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -1,9 +1,9 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,ADDR64
-# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,W64-NO-ADDR64
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,W64-NO-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64-PRE-GFX10,W64-PRE-GFX10-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64-PRE-GFX10,W64-PRE-GFX10-NO-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64
# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W32
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64,W64-NO-ADDR64
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W64
# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -verify-machineinstrs -verify-machine-dom-info --run-pass=si-fix-sgpr-copies -o - %s | FileCheck %s --check-prefixes=W32
# Test that we correctly legalize VGPR Rsrc operands in MUBUF instructions.
@@ -16,6 +16,7 @@
# TODO: S_XOR_B32_term should be `implicit-def $scc`
---
name: idxen
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -26,6 +27,48 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+
+ ; W64-PRE-GFX10-LABEL: name: idxen
+ ; W64-PRE-GFX10: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .1:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .2:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .3:
+ ; W64-PRE-GFX10-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_IDXEN]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
; W64-LABEL: name: idxen
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -38,27 +81,27 @@ body: |
; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .1:
; W64-NEXT: successors: %bb.2(0x80000000)
; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -79,27 +122,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -123,6 +166,7 @@ body: |
---
name: offen
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -133,6 +177,48 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+
+ ; W64-PRE-GFX10-LABEL: name: offen
+ ; W64-PRE-GFX10: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .1:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .2:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .3:
+ ; W64-PRE-GFX10-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFEN]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
; W64-LABEL: name: offen
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -145,27 +231,27 @@ body: |
; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .1:
; W64-NEXT: successors: %bb.2(0x80000000)
; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -186,27 +272,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -230,6 +316,7 @@ body: |
---
name: bothen
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -240,6 +327,49 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+
+
+ ; W64-PRE-GFX10-LABEL: name: bothen
+ ; W64-PRE-GFX10: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .1:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .2:
+ ; W64-PRE-GFX10-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: .3:
+ ; W64-PRE-GFX10-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_BOTHEN]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
; W64-LABEL: name: bothen
; W64: successors: %bb.1(0x80000000)
; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -252,27 +382,27 @@ body: |
; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .1:
; W64-NEXT: successors: %bb.2(0x80000000)
; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .2:
; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W64-NEXT: {{ $}}
; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W64-NEXT: {{ $}}
; W64-NEXT: .3:
@@ -293,27 +423,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[COPY1]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
@@ -337,6 +467,7 @@ body: |
---
name: addr64
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -347,28 +478,52 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-LABEL: name: addr64
- ; ADDR64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-NEXT: {{ $}}
- ; ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
- ; ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
- ; ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
- ; ADDR64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
- ; ADDR64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; ADDR64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
- ; ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
- ; ADDR64-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY6]].sub0, [[COPY1]].sub0, 0, implicit $exec
- ; ADDR64-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY6]].sub1, [[COPY1]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
- ; ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
- ; ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
- ; ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
- ; ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
- ; ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+
+ ; W64-PRE-GFX10-LABEL: name: addr64
+ ; W64-PRE-GFX10: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; W64-PRE-GFX10-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; W64-PRE-GFX10-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY6]].sub0, [[COPY1]].sub0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY6]].sub1, [[COPY1]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+ ; W64-PRE-GFX10-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
+ ; W64-PRE-GFX10-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
+ ; W64-LABEL: name: addr64
+ ; W64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; W64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; W64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 822173696
+ ; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; W64-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY6]].sub0, [[COPY1]].sub0, 0, implicit $exec
+ ; W64-NEXT: [[V_ADDC_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADDC_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADDC_U32_e64 [[COPY6]].sub1, [[COPY1]].sub1, killed [[V_ADD_CO_U32_e64_1]], 0, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, [[V_ADDC_U32_e64_]], %subreg.sub1
+ ; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
+ ; W64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
+ ; W64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
;
; W32-LABEL: name: addr64
; W32: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
@@ -409,6 +564,7 @@ body: |
---
name: offset
+tracksRegLiveness: true
liveins:
- { reg: '$vgpr0', virtual-reg: '%0' }
- { reg: '$vgpr1', virtual-reg: '%1' }
@@ -419,67 +575,110 @@ liveins:
body: |
bb.0:
liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-LABEL: name: offset
- ; ADDR64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; ADDR64-NEXT: {{ $}}
- ; ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
- ; ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
- ; ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
- ; ADDR64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
- ; ADDR64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; ADDR64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
- ; ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
- ; ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]].sub0, %subreg.sub0, [[COPY6]].sub1, %subreg.sub1
- ; ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
- ; ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
- ; ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
- ; ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+
+
+ ; W64-PRE-GFX10-ADDR64-LABEL: name: offset
+ ; W64-PRE-GFX10-ADDR64: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[S_MOV_B32_]], %subreg.sub2, [[S_MOV_B32_1]], %subreg.sub3
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]].sub0, %subreg.sub0, [[COPY6]].sub1, %subreg.sub1
+ ; W64-PRE-GFX10-ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_ADDR64_]]
+ ; W64-PRE-GFX10-ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
;
- ; W64-NO-ADDR64-LABEL: name: offset
- ; W64-NO-ADDR64: successors: %bb.1(0x80000000)
- ; W64-NO-ADDR64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
- ; W64-NO-ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
- ; W64-NO-ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; W64-NO-ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; W64-NO-ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; W64-NO-ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
- ; W64-NO-ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: .1:
- ; W64-NO-ADDR64-NEXT: successors: %bb.2(0x80000000)
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W64-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W64-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W64-NO-ADDR64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
- ; W64-NO-ADDR64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W64-NO-ADDR64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: .2:
- ; W64-NO-ADDR64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W64-NO-ADDR64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
- ; W64-NO-ADDR64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
- ; W64-NO-ADDR64-NEXT: {{ $}}
- ; W64-NO-ADDR64-NEXT: .3:
- ; W64-NO-ADDR64-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
- ; W64-NO-ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
- ; W64-NO-ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFSET]]
- ; W64-NO-ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ; W64-PRE-GFX10-NO-ADDR64-LABEL: name: offset
+ ; W64-PRE-GFX10-NO-ADDR64: successors: %bb.1(0x80000000)
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: .1:
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: successors: %bb.2(0x80000000)
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: .2:
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: {{ $}}
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: .3:
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFSET]]
+ ; W64-PRE-GFX10-NO-ADDR64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
+ ;
+ ; W64-LABEL: name: offset
+ ; W64: successors: %bb.1(0x80000000)
+ ; W64-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $sgpr30_sgpr31
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+ ; W64-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
+ ; W64-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; W64-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; W64-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; W64-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; W64-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+ ; W64-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .1:
+ ; W64-NEXT: successors: %bb.2(0x80000000)
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[PHI:%[0-9]+]]:sreg_64_xexec = PHI [[S_MOV_B64_1]], %bb.0, %15, %bb.2
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+ ; W64-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+ ; W64-NEXT: V_CMPX_EQ_U64_nosdst_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
+ ; W64-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .2:
+ ; W64-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
+ ; W64-NEXT: [[S_ANDN2_WREXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_ANDN2_WREXEC_B64 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
+ ; W64-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
+ ; W64-NEXT: {{ $}}
+ ; W64-NEXT: .3:
+ ; W64-NEXT: $exec = S_MOV_B64 [[S_MOV_B64_]]
+ ; W64-NEXT: $sgpr30_sgpr31 = COPY [[COPY]]
+ ; W64-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFSET]]
+ ; W64-NEXT: S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
;
; W32-LABEL: name: offset
; W32: successors: %bb.1(0x80000000)
@@ -493,27 +692,27 @@ body: |
; W32-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; W32-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
; W32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+ ; W32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; W32-NEXT: {{ $}}
; W32-NEXT: .1:
; W32-NEXT: successors: %bb.2(0x80000000)
; W32-NEXT: {{ $}}
+ ; W32-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.0, %15, %bb.2
; W32-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; W32-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; W32-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; W32-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; W32-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; W32-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+ ; W32-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; W32-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; W32-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .2:
; W32-NEXT: successors: %bb.1(0x40000000), %bb.3(0x40000000)
; W32-NEXT: {{ $}}
; W32-NEXT: [[BUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec
- ; W32-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+ ; W32-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 [[PHI]], implicit-def $exec, implicit-def $scc, implicit $exec
; W32-NEXT: SI_WATERFALL_LOOP %bb.1, implicit $exec
; W32-NEXT: {{ $}}
; W32-NEXT: .3:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index a69dba90a9640..a864ae1f3e1c4 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -14,25 +14,25 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: flat_load_dwordx2 v[4:5], v[6:7]
; GCN-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN-NEXT: s_mov_b32 s5, exec_lo
+; GCN-NEXT: s_mov_b32 s8, exec_lo
+; GCN-NEXT: s_mov_b32 s9, s8
; GCN-NEXT: .LBB0_2: ; Parent Loop BB0_1 Depth=1
; GCN-NEXT: ; => This Inner Loop Header: Depth=2
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_readfirstlane_b32 s8, v2
-; GCN-NEXT: v_readfirstlane_b32 s9, v3
-; GCN-NEXT: v_readfirstlane_b32 s10, v4
-; GCN-NEXT: v_readfirstlane_b32 s11, v5
-; GCN-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[2:3]
-; GCN-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[4:5]
-; GCN-NEXT: s_and_b32 s4, vcc_lo, s4
-; GCN-NEXT: s_and_saveexec_b32 s4, s4
-; GCN-NEXT: buffer_store_dword v0, v0, s[8:11], 0 offen
+; GCN-NEXT: v_readfirstlane_b32 s4, v2
+; GCN-NEXT: v_readfirstlane_b32 s5, v3
+; GCN-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
+; GCN-NEXT: v_readfirstlane_b32 s6, v4
+; GCN-NEXT: v_readfirstlane_b32 s7, v5
+; GCN-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[4:5]
+; GCN-NEXT: buffer_store_dword v0, v0, s[4:7], 0 offen
+; GCN-NEXT: s_andn2_wrexec_b32 s9, s9
; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GCN-NEXT: s_xor_b32 exec_lo, exec_lo, s4
; GCN-NEXT: s_cbranch_execnz .LBB0_2
; GCN-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
-; GCN-NEXT: s_mov_b32 exec_lo, s5
+; GCN-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GCN-NEXT: s_mov_b32 exec_lo, s8
; GCN-NEXT: s_mov_b32 vcc_lo, exec_lo
; GCN-NEXT: s_cbranch_vccnz .LBB0_1
; GCN-NEXT: ; %bb.4: ; %DummyReturnBlock
@@ -46,26 +46,25 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB0_2 Depth 2
; GFX11-NEXT: flat_load_b128 v[2:5], v[0:1]
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: s_mov_b32 s4, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s5, s4
; GFX11-NEXT: .LBB0_2: ; Parent Loop BB0_1 Depth=1
; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v2
-; GFX11-NEXT: v_readfirstlane_b32 s5, v3
-; GFX11-NEXT: v_readfirstlane_b32 s6, v4
-; GFX11-NEXT: v_readfirstlane_b32 s7, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
-; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-NEXT: buffer_store_b32 v0, v0, s[4:7], 0 offen
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: v_readfirstlane_b32 s1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[2:3]
+; GFX11-NEXT: v_readfirstlane_b32 s2, v4
+; GFX11-NEXT: v_readfirstlane_b32 s3, v5
+; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[4:5]
+; GFX11-NEXT: buffer_store_b32 v0, v0, s[0:3], 0 offen
+; GFX11-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
-; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
; GFX11-NEXT: ; %bb.3: ; in Loop: Header=BB0_1 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 exec_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_mov_b32 vcc_lo, exec_lo
; GFX11-NEXT: s_cbranch_vccnz .LBB0_1
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 3bc67562012e5..d7c46d71fd016 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -233,9 +233,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %16:vgpr_32, %bb.0, %4, %bb.9
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %48:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %50:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %52:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %50:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -249,24 +249,24 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %54:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %56:vgpr_32, %bb.4, [[PHI1]], %bb.2
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %41, %bb.4
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI undef %58:vgpr_32, %bb.4, [[PHI1]], %bb.2
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], killed [[PHI4]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], killed [[PHI5]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY6]]
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI5]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI6]]
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI4]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -286,24 +286,24 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.7:
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %58:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %60:vgpr_32, %bb.8, [[COPY4]], %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI7:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %33, %bb.8
+ ; SI-NEXT: [[PHI8:%[0-9]+]]:vreg_64 = PHI undef %60:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[PHI9:%[0-9]+]]:vgpr_32 = PHI undef %62:vgpr_32, %bb.8, [[COPY4]], %bb.6
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], killed [[PHI6]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], killed [[PHI8]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY9]]
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI7]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI9]]
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI7]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -314,9 +314,9 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.10.end:
- ; SI-NEXT: [[PHI8:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
+ ; SI-NEXT: [[PHI10:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
; SI-NEXT: SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; SI-NEXT: $vgpr0 = COPY killed [[PHI8]]
+ ; SI-NEXT: $vgpr0 = COPY killed [[PHI10]]
; SI-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
main_body:
%cc = icmp sgt i32 %z, 5
@@ -356,8 +356,8 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %16:vgpr_32, %bb.0, %4, %bb.9
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %49:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %51:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %51:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %53:vgpr_32, %bb.9
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -371,12 +371,12 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %53:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI3]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI3]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI3:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %42, %bb.4
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], killed [[PHI3]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE1]], killed [[PHI4]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
@@ -387,7 +387,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.3, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
@@ -407,12 +407,12 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.7:
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
- ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %34, %bb.8
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
+ ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], killed [[PHI4]], implicit $exec
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[V_CMP_EQ_U64_e64_1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 [[REG_SEQUENCE3]], killed [[PHI6]], implicit-def $exec, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.8:
; SI-NEXT: successors: %bb.7(0x40000000), %bb.9(0x40000000)
@@ -423,7 +423,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE3]], 0, csr_amdgpu_si_gfx, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed $vgpr0, implicit-def $vgpr0
; SI-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; SI-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI5]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.7, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.9:
@@ -434,9 +434,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: S_BRANCH %bb.1
; SI-NEXT: {{ $}}
; SI-NEXT: bb.10.end:
- ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
+ ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[COPY8]], %bb.5
; SI-NEXT: SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; SI-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI5]], 0, killed [[COPY4]], 0, 0, implicit $mode, implicit $exec
+ ; SI-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI7]], 0, killed [[COPY4]], 0, 0, implicit $mode, implicit $exec
; SI-NEXT: $vgpr0 = COPY killed [[V_ADD_F32_e64_]]
; SI-NEXT: SI_RETURN_TO_EPILOG killed $vgpr0
main_body:
@@ -582,27 +582,24 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: bb.2:
; SI-NEXT: successors: %bb.3(0x80000000)
; SI-NEXT: {{ $}}
+ ; SI-NEXT: [[PHI:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.1, %38, %bb.6
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
- ; SI-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[V_CMP_EQ_U64_e64_]], killed [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub4, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub5, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_4]], %subreg.sub0, [[V_READFIRSTLANE_B32_5]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit $exec
- ; SI-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[S_AND_B32_]], killed [[V_CMP_EQ_U64_e64_2]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub4_sub5, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub6, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub7, implicit $exec
; SI-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_6]], %subreg.sub0, [[V_READFIRSTLANE_B32_7]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit $exec
- ; SI-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[S_AND_B32_1]], killed [[V_CMP_EQ_U64_e64_3]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]].sub6_sub7, implicit-def $exec, implicit $exec
; SI-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sgpr_256 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_2]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub3, killed [[V_READFIRSTLANE_B32_4]], %subreg.sub4, killed [[V_READFIRSTLANE_B32_5]], %subreg.sub5, killed [[V_READFIRSTLANE_B32_6]], %subreg.sub6, killed [[V_READFIRSTLANE_B32_7]], %subreg.sub7
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_2]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.3:
; SI-NEXT: successors: %bb.4(0x80000000)
@@ -612,30 +609,29 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
; SI-NEXT: bb.4:
; SI-NEXT: successors: %bb.5(0x80000000)
; SI-NEXT: {{ $}}
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.3, %55, %bb.5
; SI-NEXT: [[V_READFIRSTLANE_B32_8:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_9:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_8]], %subreg.sub0, [[V_READFIRSTLANE_B32_9]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_4:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE6]], [[GLOBAL_LOAD_DWORDX4_2]].sub0_sub1, implicit $exec
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE6]], [[GLOBAL_LOAD_DWORDX4_2]].sub0_sub1, implicit-def $exec, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_10:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub2, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_11:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[GLOBAL_LOAD_DWORDX4_2]].sub3, implicit $exec
; SI-NEXT: [[REG_SEQUENCE7:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_10]], %subreg.sub0, [[V_READFIRSTLANE_B32_11]], %subreg.sub1
- ; SI-NEXT: [[V_CMP_EQ_U64_e64_5:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 killed [[REG_SEQUENCE7]], [[GLOBAL_LOAD_DWORDX4_2]].sub2_sub3, implicit $exec
- ; SI-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 killed [[V_CMP_EQ_U64_e64_4]], killed [[V_CMP_EQ_U64_e64_5]], implicit-def dead $scc
+ ; SI-NEXT: V_CMPX_EQ_U64_nosdst_e32 killed [[REG_SEQUENCE7]], [[GLOBAL_LOAD_DWORDX4_2]].sub2_sub3, implicit-def $exec, implicit $exec
; SI-NEXT: [[REG_SEQUENCE8:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_8]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_9]], %subreg.sub1, killed [[V_READFIRSTLANE_B32_10]], %subreg.sub2, killed [[V_READFIRSTLANE_B32_11]], %subreg.sub3
- ; SI-NEXT: [[S_AND_SAVEEXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_3]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.5:
; SI-NEXT: successors: %bb.4(0x40000000), %bb.6(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[IMAGE_SAMPLE_V1_V2_nsa_gfx10_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_nsa_gfx10 undef %29:vgpr_32, undef %31:vgpr_32, [[REG_SEQUENCE5]], killed [[REG_SEQUENCE8]], 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.4, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.6:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.7(0x40000000)
; SI-NEXT: {{ $}}
; SI-NEXT: $exec_lo = S_MOV_B32 killed [[S_MOV_B32_1]]
- ; SI-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
+ ; SI-NEXT: [[S_ANDN2_WREXEC_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_ANDN2_WREXEC_B32 killed [[PHI]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; SI-NEXT: {{ $}}
; SI-NEXT: bb.7:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
index b46f5f5640b66..9e0db1dfa26c5 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange.ll
@@ -177,18 +177,19 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_cbranch_execz .LBB3_4
; SI-NEXT: ; %bb.1: ; %else
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB3_2: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_readfirstlane_b32 s5, v5
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB3_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -199,18 +200,19 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: s_cbranch_execz .LBB3_8
; SI-NEXT: ; %bb.5: ; %if
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB3_6: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: v_readfirstlane_b32 s5, v3
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
; SI-NEXT: v_mov_b32_e32 v1, v0
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
; SI-NEXT: ; implicit-def: $vgpr0
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB3_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -254,17 +256,18 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_cbranch_execz .LBB4_4
; SI-NEXT: ; %bb.1: ; %else
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB4_2: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v4
; SI-NEXT: v_readfirstlane_b32 s5, v5
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[4:5]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[4:5]
; SI-NEXT: v_mov_b32_e32 v0, v40
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr4_vgpr5
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB4_2
; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 exec_lo, s7
@@ -274,17 +277,18 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: s_cbranch_execz .LBB4_8
; SI-NEXT: ; %bb.5: ; %if
; SI-NEXT: s_mov_b32 s7, exec_lo
+; SI-NEXT: s_mov_b32 s8, s7
; SI-NEXT: .LBB4_6: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_readfirstlane_b32 s4, v2
; SI-NEXT: v_readfirstlane_b32 s5, v3
-; SI-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
-; SI-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; SI-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; SI-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[2:3]
; SI-NEXT: v_mov_b32_e32 v0, v40
; SI-NEXT: s_mov_b64 s[0:1], s[12:13]
; SI-NEXT: s_mov_b64 s[2:3], s[14:15]
; SI-NEXT: s_swappc_b64 s[30:31], s[4:5]
+; SI-NEXT: s_andn2_wrexec_b32 s8, s8
; SI-NEXT: ; implicit-def: $vgpr2_vgpr3
-; SI-NEXT: s_xor_b32 exec_lo, exec_lo, s8
; SI-NEXT: s_cbranch_execnz .LBB4_6
; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 exec_lo, s7
More information about the llvm-commits
mailing list