[llvm] [AMDGPU] Refactor GFX11 VALU Mask Hazard Waitcnt Merging (PR #169213)
Carl Ritson via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 03:09:54 PDT 2026
https://github.com/perlfu updated https://github.com/llvm/llvm-project/pull/169213
>From 62203e1626776fbffb0dac901f1685b77c488e59 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 26 Nov 2025 16:00:11 +0900
Subject: [PATCH 1/2] [AMDGPU] Refactor GFX11 VALU Mask Hazard Waitcnt Merging
Move GFX11 SGPR VALU mask hazard waitcnt merging to a forward
scan within the AMDGPUWaitSGPRHazard pass.
This simplifies the hazard recognizer code and allows merging of
waitcnt instructions in cases where SGPRs unaffected by pending
writes are used.
In turn this greatly decreasing numbers of waits inserted in
sequences of V_CMP instructions writing SGPRs improving VALU
pipeline performance.
---
.../Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 120 ++++++++++++++-
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 99 +++---------
.../atomic_optimizations_global_pointer.ll | 68 ++++-----
.../atomic_optimizations_local_pointer.ll | 116 +++++++-------
.../atomic_optimizations_pixelshader.ll | 2 +-
.../AMDGPU/gfx11-sgpr-hazard-latency.mir | 3 +-
.../global-saddr-atomics-min-max-system.ll | 16 +-
.../AMDGPU/global_atomics_scan_fadd.ll | 50 +++----
.../AMDGPU/global_atomics_scan_fmax.ll | 33 ++--
.../AMDGPU/global_atomics_scan_fmin.ll | 33 ++--
.../AMDGPU/global_atomics_scan_fsub.ll | 50 +++----
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll | 48 +++---
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll | 46 +++---
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll | 4 +-
.../CodeGen/AMDGPU/valu-mask-write-hazard.mir | 141 ++++++++++++++++--
25 files changed, 528 insertions(+), 357 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index b1491985c5d6b..f54958227d04b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -63,6 +63,7 @@ class AMDGPUWaitSGPRHazards {
case AMDGPU::EXEC_HI:
case AMDGPU::SGPR_NULL:
case AMDGPU::SGPR_NULL64:
+ case AMDGPU::SCC:
return {};
default:
break;
@@ -437,9 +438,121 @@ class AMDGPUWaitSGPRHazards {
return Changed;
}
+ bool runWaitMerging(MachineFunction &MF) {
+ // Perform per-block merging of existing s_waitcnt_depctr instructions.
+ // Track set of SGPR writes before a given wait instruction, and search
+ // for reads of these SGPRs.
+ // Move the wait to just before the read to improve pipelining.
+ // If no related reads occur before subsequent wait then merged waits.
+ const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
+ AMDGPU::DepCtr::encodeFieldVaSdst(
+ AMDGPU::DepCtr::encodeFieldVaVcc(0, *ST), 0),
+ 0);
+ const unsigned VccLoIdx = *sgprNumber(AMDGPU::VCC_LO, *TRI);
+ const unsigned VccHiIdx = *sgprNumber(AMDGPU::VCC_HI, *TRI);
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ std::bitset<128> WriteSet, PendingSALUWriteSet, PendingVALUWriteSet;
+ MachineInstr *PrevWait = nullptr;
+
+ auto CommitWrites = [&](unsigned Mask) {
+ if (!AMDGPU::DepCtr::decodeFieldSaSdst(Mask))
+ WriteSet |= PendingSALUWriteSet;
+ bool VccLoBit = WriteSet[VccLoIdx];
+ bool VccHiBit = WriteSet[VccHiIdx];
+ if (!AMDGPU::DepCtr::decodeFieldVaSdst(Mask)) {
+ WriteSet |= PendingVALUWriteSet;
+ WriteSet.set(VccLoIdx, VccLoBit);
+ WriteSet.set(VccLoIdx, VccHiBit);
+ }
+ if (!AMDGPU::DepCtr::decodeFieldVaVcc(Mask)) {
+ WriteSet.set(VccLoIdx, VccLoBit | PendingVALUWriteSet[VccLoIdx]);
+ WriteSet.set(VccHiIdx, VccHiBit | PendingVALUWriteSet[VccHiIdx]);
+ }
+ // Clear all pending writes
+ PendingSALUWriteSet.reset();
+ PendingVALUWriteSet.reset();
+ };
+
+ for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
+ E = MBB.instr_end();
+ MI != E; ++MI) {
+ if (MI->isMetaInstruction())
+ continue;
+
+ if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && !MI->isBundled() &&
+ (MI->getOperand(0).getImm() & ConstantMaskBits) ==
+ ConstantMaskBits) {
+ if (PrevWait) {
+ // Merge previous wait into this one.
+ MachineOperand &MaskOp = MI->getOperand(0);
+ MaskOp.setImm(
+ mergeMasks(PrevWait->getOperand(0).getImm(), MaskOp.getImm()));
+ PrevWait->eraseFromParent();
+ Changed = true;
+ } else {
+ // Starting a new region using fresh write set.
+ WriteSet.reset();
+ }
+ CommitWrites(MI->getOperand(0).getImm());
+ PrevWait = &*MI;
+ continue;
+ }
+
+ // Do not optimize over branches
+ if (PrevWait && (MI->isCall() || MI->isReturn() || MI->isBranch())) {
+ PrevWait->moveBefore(&*MI);
+ PrevWait = nullptr;
+ Changed = true;
+ }
+
+ const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
+ const bool IsSALU = SIInstrInfo::isSALU(*MI);
+ if (!IsVALU && !IsSALU)
+ continue;
+
+ for (const MachineOperand &Op : MI->operands()) {
+ if (!Op.isReg())
+ continue;
+ Register Reg = Op.getReg();
+ if (!TRI->isSGPRReg(*MRI, Reg))
+ continue;
+
+ auto RegNumber = sgprNumber(Reg, *TRI);
+ if (!RegNumber)
+ continue;
+ unsigned RegN = *RegNumber;
+
+ uint8_t SGPRCount =
+ AMDGPU::getRegBitWidth(*TRI->getRegClassForReg(*MRI, Reg)) / 32;
+
+ if (Op.isDef()) {
+ for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
+ if (IsSALU)
+ PendingSALUWriteSet.set(RegN + RegIdx);
+ else
+ PendingVALUWriteSet.set(RegN + RegIdx);
+ }
+ continue;
+ }
+
+ for (uint8_t RegIdx = 0; RegIdx < SGPRCount && PrevWait; ++RegIdx) {
+ if (!WriteSet[RegN + RegIdx])
+ continue;
+ // Move the wait to here, the last point it can be valid
+ PrevWait->moveBefore(&*MI);
+ PrevWait = nullptr;
+ Changed = true;
+ }
+ }
+ }
+ }
+ return Changed;
+ }
+
bool run(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
- if (!ST->hasVALUReadSGPRHazard())
+ if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
return false;
// Parse settings
@@ -464,6 +577,11 @@ class AMDGPUWaitSGPRHazards {
MRI = &MF.getRegInfo();
DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
+ // VALU mask write hazards have already been handled, but this pass
+ // performs a forward scan to optimize them.
+ if (ST->hasVALUMaskWriteHazard())
+ return ST->isWave64() ? runWaitMerging(MF) : false;
+
auto CallingConv = MF.getFunction().getCallingConv();
if (!AMDGPU::isEntryFunctionCC(CallingConv) &&
!CullSGPRHazardsOnFunctionBoundary) {
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 722ddb5d0c4dd..c06e4ec409c3b 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -3556,15 +3556,14 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
};
SmallVector<const MachineInstr *> WaitInstrs;
- bool HasSGPRRead = false;
StateType InitialState;
// Look for SGPR write.
MachineOperand *HazardDef = nullptr;
- for (MachineOperand &Op : MI->operands()) {
+ for (MachineOperand &Op : MI->all_defs()) {
if (!Op.isReg())
continue;
- if (Op.isDef() && HazardDef)
+ if (HazardDef)
continue;
Register Reg = Op.getReg();
@@ -3576,11 +3575,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
if (!TRI->isSGPRReg(MRI, Reg))
continue;
}
- // Also check for SGPR reads.
- if (Op.isUse()) {
- HasSGPRRead = true;
- continue;
- }
assert(!HazardDef);
HazardDef = &Op;
@@ -3644,49 +3638,31 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
}
};
- const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
- AMDGPU::DepCtr::encodeFieldVaSdst(AMDGPU::DepCtr::encodeFieldVaVcc(0, ST),
- 0),
- 0);
auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
- switch (I.getOpcode()) {
- case AMDGPU::S_WAITCNT_DEPCTR:
- // Record mergable waits within region of instructions free of SGPR reads.
- if (!HasSGPRRead && I.getParent() == MI->getParent() && !I.isBundled() &&
- (I.getOperand(0).getImm() & ConstantMaskBits) == ConstantMaskBits)
- WaitInstrs.push_back(&I);
- break;
- default:
- // Update tracking of SGPR reads and writes.
- for (auto &Op : I.operands()) {
- if (!Op.isReg())
- continue;
+ // Update tracking of SGPR writes.
+ for (auto &Op : I.all_defs()) {
+ if (!Op.isReg())
+ continue;
- Register Reg = Op.getReg();
- if (IgnoreableSGPR(Reg))
+ Register Reg = Op.getReg();
+ if (IgnoreableSGPR(Reg))
+ continue;
+ if (!IsVCC(Reg)) {
+ if (Op.isImplicit())
continue;
- if (!IsVCC(Reg)) {
- if (Op.isImplicit())
- continue;
- if (!TRI->isSGPRReg(MRI, Reg))
- continue;
- }
- if (Op.isUse()) {
- HasSGPRRead = true;
+ if (!TRI->isSGPRReg(MRI, Reg))
continue;
- }
+ }
- // Stop tracking any SGPRs with writes on the basis that they will
- // already have an appropriate wait inserted afterwards.
- SmallVector<Register, 2> Found;
- for (Register SGPR : State.HazardSGPRs) {
- if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
- Found.push_back(SGPR);
- }
- for (Register SGPR : Found)
- State.HazardSGPRs.erase(SGPR);
+ // Stop tracking any SGPRs with writes on the basis that they will
+ // already have an appropriate wait inserted afterwards.
+ SmallVector<Register, 2> Found;
+ for (Register SGPR : State.HazardSGPRs) {
+ if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
+ Found.push_back(SGPR);
}
- break;
+ for (Register SGPR : Found)
+ State.HazardSGPRs.erase(SGPR);
}
};
@@ -3702,39 +3678,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
: AMDGPU::DepCtr::encodeFieldVaSdst(0, ST))
: AMDGPU::DepCtr::encodeFieldSaSdst(0, ST);
- // Try to merge previous waits into this one for regions with no SGPR reads.
- if (!WaitInstrs.empty()) {
- // Note: WaitInstrs contains const pointers, so walk backward from MI to
- // obtain a mutable pointer to each instruction to be merged.
- // This is expected to be a very short walk within the same block.
- SmallVector<MachineInstr *> ToErase;
- unsigned Found = 0;
- for (MachineBasicBlock::reverse_iterator It = MI->getReverseIterator(),
- End = MI->getParent()->rend();
- Found < WaitInstrs.size() && It != End; ++It) {
- MachineInstr *WaitMI = &*It;
- // Find next wait instruction.
- if (std::as_const(WaitMI) != WaitInstrs[Found])
- continue;
- Found++;
- unsigned WaitMask = WaitMI->getOperand(0).getImm();
- assert((WaitMask & ConstantMaskBits) == ConstantMaskBits);
- DepCtr = AMDGPU::DepCtr::encodeFieldSaSdst(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldSaSdst(WaitMask),
- AMDGPU::DepCtr::decodeFieldSaSdst(DepCtr)));
- DepCtr = AMDGPU::DepCtr::encodeFieldVaSdst(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldVaSdst(WaitMask),
- AMDGPU::DepCtr::decodeFieldVaSdst(DepCtr)));
- DepCtr = AMDGPU::DepCtr::encodeFieldVaVcc(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldVaVcc(WaitMask),
- AMDGPU::DepCtr::decodeFieldVaVcc(DepCtr)));
- ToErase.push_back(WaitMI);
- }
- assert(Found == WaitInstrs.size());
- for (MachineInstr *WaitMI : ToErase)
- WaitMI->eraseFromParent();
- }
-
// Add s_waitcnt_depctr after SGPR write.
auto NextMI = std::next(MI->getIterator());
auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 25991d274a278..0c420cdbb108e 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -3596,44 +3596,44 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
@@ -3685,10 +3685,10 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32 v8, vcc, s2, v10
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v9, null, s3, v11, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -8743,44 +8743,44 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
@@ -8855,10 +8855,10 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v10
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e64 v9, null, s3, v11, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -15219,8 +15219,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -15276,8 +15276,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -16519,8 +16519,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
@@ -16560,7 +16560,6 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_add_f32_e32 v2, s13, v2
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX1164-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
@@ -16568,8 +16567,9 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX1164-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
@@ -16580,8 +16580,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 7a5f9c86a2326..b813e5b450e0a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1080,10 +1080,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -1625,10 +1625,10 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3074,50 +3074,49 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -3132,6 +3131,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0
@@ -3142,9 +3142,9 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -3908,29 +3908,29 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
; GFX1164_DPP-NEXT: v_permlane64_b32 v3, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_permlane64_b32 v4, v1
@@ -5196,10 +5196,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -5741,10 +5741,10 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -7221,50 +7221,49 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -7279,6 +7278,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0
@@ -7289,9 +7289,9 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB14_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -8087,10 +8087,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8941,7 +8941,6 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -8954,6 +8953,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -9725,10 +9725,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10551,7 +10551,6 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -10564,6 +10563,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -11335,10 +11335,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12161,7 +12161,6 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -12174,6 +12173,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -12945,10 +12945,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -14284,7 +14284,6 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -14299,6 +14298,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -14309,9 +14309,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB23_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -15150,10 +15150,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -16489,7 +16489,6 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -16504,6 +16503,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -16514,9 +16514,9 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB26_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -17355,10 +17355,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -18684,7 +18684,6 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -18699,6 +18698,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -18709,9 +18709,9 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB29_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -19542,10 +19542,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -20870,7 +20870,6 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -20885,6 +20884,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -20895,9 +20895,9 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB32_2
; GFX1164_DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 367d38aaff89d..4c87e6936e314 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -626,10 +626,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_or_saveexec_b64 s[10:11], -1
-; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: v_readlane_b32 s12, v1, 63
; GFX1164-NEXT: v_readlane_b32 s14, v1, 47
; GFX1164-NEXT: v_writelane_b32 v3, s13, 32
+; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: s_mov_b64 exec, s[10:11]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
index 35e4122f3ff72..f969b3a30996d 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
+++ b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
@@ -42,13 +42,12 @@ body: |
; GFX11-NEXT: renamable $sgpr17 = S_CSELECT_B32 killed renamable $sgpr17, 0, implicit killed $scc
; GFX11-NEXT: renamable $vgpr7 = V_ADD_U32_e32 64, $vgpr5, implicit $exec
; GFX11-NEXT: renamable $sgpr18_sgpr19 = V_CMP_GT_U32_e64 $sgpr15, $vgpr5, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: renamable $vgpr6 = V_ADD_U32_e32 $sgpr8, $vgpr5, implicit $exec
; GFX11-NEXT: renamable $vgpr8 = V_ADD_U32_e32 $sgpr9, killed $vgpr5, implicit $exec
; GFX11-NEXT: renamable $sgpr16 = nsw S_ADD_I32 killed renamable $sgpr16, -1, implicit-def dead $scc
; GFX11-NEXT: renamable $sgpr20_sgpr21 = V_CMP_GT_U32_e64 $sgpr15, $vgpr7, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: S_CMP_LG_U32 renamable $sgpr16, 0, implicit-def $scc
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: renamable $vgpr5 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr6, $sgpr18_sgpr19, implicit $exec
; GFX11-NEXT: renamable $vgpr6 = V_ADD_U32_e32 $sgpr8, $vgpr7, implicit $exec
; GFX11-NEXT: renamable $vgpr7 = V_ADD_U32_e32 $sgpr9, killed $vgpr7, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
index 63d63da56080b..329869f05ea5f 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
@@ -769,9 +769,9 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -884,9 +884,9 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -1693,9 +1693,9 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -1808,9 +1808,9 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -2617,9 +2617,9 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -2732,9 +2732,9 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -3541,9 +3541,9 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -3656,9 +3656,9 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index f1ecc94398192..690ba81c6a4d2 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -1063,14 +1063,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2290,14 +2290,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -3577,14 +3577,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4360,14 +4360,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -5672,14 +5672,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -7019,11 +7019,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -8460,11 +8460,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -9901,11 +9901,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -10824,11 +10824,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -12265,11 +12265,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 92c54d4c20252..7793e941a07d1 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -907,14 +907,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -1886,14 +1886,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2865,14 +2865,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4022,15 +4022,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5219,15 +5218,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6416,15 +6414,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index a59a1d518022b..006ca88d65924 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -907,14 +907,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -1886,14 +1886,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2865,14 +2865,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4022,15 +4022,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5219,15 +5218,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6416,15 +6414,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 8aeffd7fae1a6..b8a3714c12123 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -1149,14 +1149,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2488,14 +2488,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -3827,14 +3827,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4662,14 +4662,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -6000,14 +6000,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -7347,11 +7347,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -8787,11 +8787,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -10228,11 +10228,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -11151,11 +11151,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -12591,11 +12591,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
index 08645068ac4cf..0868d633cb55e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
@@ -2333,46 +2333,46 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
@@ -2380,13 +2380,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2424,46 +2424,46 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
@@ -2471,13 +2471,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index 6b7d8ee27bd98..9249375e5c8c1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -2098,13 +2098,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2185,13 +2185,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
index a257d86f538c2..3738d32a7070e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
@@ -1575,11 +1575,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1619,11 +1619,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2355,12 +2355,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2431,12 +2431,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
index f1a77a27e4a97..fb4cd5f29962f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
@@ -1475,11 +1475,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1519,11 +1519,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2255,12 +2255,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2331,12 +2331,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
index bd0f6ea162663..d927bd18b2eda 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
@@ -1475,11 +1475,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1519,11 +1519,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2255,12 +2255,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2331,12 +2331,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
index bdf182e90926a..8462410a95b4d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
@@ -1581,11 +1581,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1626,11 +1626,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2384,12 +2384,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2461,12 +2461,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
index 132612c1f5e6f..68703a6912f76 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
@@ -2197,13 +2197,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2291,13 +2291,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
index c19059b754831..2eff1b93892f9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
@@ -2197,13 +2197,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2291,13 +2291,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index 740b83329ad44..63e8b4a74baee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -2098,13 +2098,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2185,13 +2185,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
index 1f51a317cdc08..5d3202f428787 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
@@ -2420,51 +2420,50 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_sub_u32 s0, 0, s2
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_subb_u32 s1, 0, s3
; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2514,51 +2513,50 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_sub_u32 s0, 0, s2
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_subb_u32 s1, 0, s3
; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index 1684aa599344d..a5d60521ca9e1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -2201,13 +2201,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2295,13 +2295,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 29d45ce160a71..ee1ee34c2abcb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -2189,13 +2189,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2283,13 +2283,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 77d89e18dbadd..add382f8e0b27 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -2360,13 +2360,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2447,13 +2447,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index 8b758a75df9c3..3c9faaa7f1940 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
--- |
@@ -57,6 +57,9 @@
define amdgpu_gs void @mask_hazard_combine3() { ret void }
define amdgpu_gs void @mask_hazard_combine4() { ret void }
define amdgpu_gs void @mask_hazard_combine5() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize1() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize2() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize3() { ret void }
...
---
@@ -785,7 +788,6 @@ body: |
; GFX11-LABEL: name: mask_hazard_partial_cancel2
; GFX11: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
; GFX11-NEXT: $vcc_hi = S_MOV_B32 0
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
; GFX11-NEXT: $sgpr0 = S_MOV_B32 $vcc_lo
; GFX11-NEXT: $vcc = S_MOV_B64 1
; GFX11-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
@@ -1020,10 +1022,9 @@ body: |
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
; GFX11-NEXT: $sgpr0 = S_MOV_B32 0
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0_SaSdst_0
; GFX11-NEXT: $sgpr1 = S_MOV_B32 $sgpr4
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $vgpr5, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_SaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0_SaSdst_0
; GFX11-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: mask_hazard_combine2
@@ -1054,11 +1055,9 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: mask_hazard_combine3
@@ -1087,9 +1086,8 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: $sgpr4_sgpr5 = S_MOV_B64 $vcc
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
@@ -1123,9 +1121,8 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr1
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
@@ -1149,3 +1146,125 @@ body: |
$sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
S_ENDPGM 0
...
+
+---
+name: mask_hazard_optimize1
+body: |
+ bb.0:
+ ; GCN-LABEL: name: mask_hazard_optimize1
+ ; GCN: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GCN-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GCN-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GCN-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GCN-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GCN-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GCN-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ $sgpr4 = S_MOV_B32 $sgpr1
+ $sgpr5 = S_MOV_B32 $sgpr2
+ $sgpr6 = S_MOV_B32 $sgpr3
+ $sgpr7 = S_MOV_B32 $sgpr4
+ $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: mask_hazard_optimize2
+body: |
+ bb.0:
+ ; GFX11-LABEL: name: mask_hazard_optimize2
+ ; GFX11: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX11-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ ; GFX11-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GFX11-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: mask_hazard_optimize2
+ ; GFX12: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GFX12-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX12-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ ; GFX12-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
+ ; GFX12-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GFX12-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ $sgpr4 = S_MOV_B32 $sgpr1
+ $sgpr5 = S_MOV_B32 $sgpr2
+ $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ $sgpr6 = S_MOV_B32 $sgpr3
+ $sgpr7 = S_MOV_B32 $sgpr4
+ $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: mask_hazard_optimize3
+body: |
+ bb.0:
+ ; GFX11-LABEL: name: mask_hazard_optimize3
+ ; GFX11: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ ; GFX11-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX11-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ ; GFX11-NEXT: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ ; GFX11-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ ; GFX11-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ ; GFX11-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX11-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+ ;
+ ; GFX12-LABEL: name: mask_hazard_optimize3
+ ; GFX12: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX12-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ ; GFX12-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ ; GFX12-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ ; GFX12-NEXT: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ ; GFX12-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ ; GFX12-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ ; GFX12-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX12-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+ V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ S_WAITCNT_DEPCTR 65533
+ $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+...
>From efdd16f6b874b9ca9361444a3cedc1a7236e8373 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Tue, 14 Jul 2026 18:45:13 +0900
Subject: [PATCH 2/2] - Address reviewer comments
---
.../Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 35 +++++++++----------
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 3 +-
2 files changed, 17 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index f54958227d04b..378ccf4d738ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -17,6 +17,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallBitVector.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -452,7 +453,7 @@ class AMDGPUWaitSGPRHazards {
const unsigned VccHiIdx = *sgprNumber(AMDGPU::VCC_HI, *TRI);
bool Changed = false;
for (MachineBasicBlock &MBB : MF) {
- std::bitset<128> WriteSet, PendingSALUWriteSet, PendingVALUWriteSet;
+ SmallBitVector WriteSet(128), PendingSALUWriteSet(128), PendingVALUWriteSet(128);
MachineInstr *PrevWait = nullptr;
auto CommitWrites = [&](unsigned Mask) {
@@ -461,26 +462,25 @@ class AMDGPUWaitSGPRHazards {
bool VccLoBit = WriteSet[VccLoIdx];
bool VccHiBit = WriteSet[VccHiIdx];
if (!AMDGPU::DepCtr::decodeFieldVaSdst(Mask)) {
+ // Apply pending VALU set minus VCC bits
WriteSet |= PendingVALUWriteSet;
- WriteSet.set(VccLoIdx, VccLoBit);
- WriteSet.set(VccLoIdx, VccHiBit);
+ WriteSet[VccLoIdx] = VccLoBit;
+ WriteSet[VccHiIdx] = VccHiBit;
}
if (!AMDGPU::DepCtr::decodeFieldVaVcc(Mask)) {
- WriteSet.set(VccLoIdx, VccLoBit | PendingVALUWriteSet[VccLoIdx]);
- WriteSet.set(VccHiIdx, VccHiBit | PendingVALUWriteSet[VccHiIdx]);
+ WriteSet[VccLoIdx] = VccLoBit || PendingVALUWriteSet[VccLoIdx];
+ WriteSet[VccHiIdx] = VccHiBit || PendingVALUWriteSet[VccHiIdx];
}
// Clear all pending writes
PendingSALUWriteSet.reset();
PendingVALUWriteSet.reset();
};
- for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
- E = MBB.instr_end();
- MI != E; ++MI) {
+ for (auto MI = MBB.begin(), E = MBB.end(); MI != E; ++MI) {
if (MI->isMetaInstruction())
continue;
- if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && !MI->isBundled() &&
+ if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
(MI->getOperand(0).getImm() & ConstantMaskBits) ==
ConstantMaskBits) {
if (PrevWait) {
@@ -506,7 +506,7 @@ class AMDGPUWaitSGPRHazards {
Changed = true;
}
- const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
+ const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/false);
const bool IsSALU = SIInstrInfo::isSALU(*MI);
if (!IsVALU && !IsSALU)
continue;
@@ -527,18 +527,15 @@ class AMDGPUWaitSGPRHazards {
AMDGPU::getRegBitWidth(*TRI->getRegClassForReg(*MRI, Reg)) / 32;
if (Op.isDef()) {
- for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
- if (IsSALU)
- PendingSALUWriteSet.set(RegN + RegIdx);
- else
- PendingVALUWriteSet.set(RegN + RegIdx);
- }
+ if (IsSALU)
+ PendingSALUWriteSet.set(RegN, RegN + SGPRCount);
+ else
+ PendingVALUWriteSet.set(RegN, RegN + SGPRCount);
continue;
}
- for (uint8_t RegIdx = 0; RegIdx < SGPRCount && PrevWait; ++RegIdx) {
- if (!WriteSet[RegN + RegIdx])
- continue;
+ if (PrevWait &&
+ WriteSet.find_prev(RegN + SGPRCount) >= (signed)RegN) {
// Move the wait to here, the last point it can be valid
PrevWait->moveBefore(&*MI);
PrevWait = nullptr;
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index c06e4ec409c3b..a0002bbb07880 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -3564,7 +3564,7 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
if (!Op.isReg())
continue;
if (HazardDef)
- continue;
+ break;
Register Reg = Op.getReg();
if (IgnoreableSGPR(Reg))
@@ -3576,7 +3576,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
continue;
}
- assert(!HazardDef);
HazardDef = &Op;
}
More information about the llvm-commits
mailing list