[llvm] f2ef567 - [AMDGPU] Refactor GFX11 VALU Mask Hazard Waitcnt Merging (#169213)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 22:23:08 PDT 2026
Author: Carl Ritson
Date: 2026-07-17T14:23:03+09:00
New Revision: f2ef5670cf70371153caeb0ebbeae2323a61f8e0
URL: https://github.com/llvm/llvm-project/commit/f2ef5670cf70371153caeb0ebbeae2323a61f8e0
DIFF: https://github.com/llvm/llvm-project/commit/f2ef5670cf70371153caeb0ebbeae2323a61f8e0.diff
LOG: [AMDGPU] Refactor GFX11 VALU Mask Hazard Waitcnt Merging (#169213)
Move GFX11 SGPR VALU mask hazard waitcnt merging to the a forward scan
within the AMDGPUWaitSGPRHazard pass.
This simplifies the hazard recognizer code and allows merging of waitcnt
instructions in cases where SGPRs unaffected by pending writes are used.
In turn this greatly decreasing numbers of waits inserted in sequences
of V_CMP instructions writing SGPRs improving VALU pipeline performance.
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
llvm/test/CodeGen/AMDGPU/atomic_optimizations_dpp_lds_expected_active_lanes.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index b1491985c5d6b..51860f195cce7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -17,6 +17,7 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallBitVector.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
using namespace llvm;
@@ -63,6 +64,7 @@ class AMDGPUWaitSGPRHazards {
case AMDGPU::EXEC_HI:
case AMDGPU::SGPR_NULL:
case AMDGPU::SGPR_NULL64:
+ case AMDGPU::SCC:
return {};
default:
break;
@@ -437,9 +439,117 @@ class AMDGPUWaitSGPRHazards {
return Changed;
}
+ bool runWaitMerging(MachineFunction &MF) {
+ // Perform per-block merging of existing s_waitcnt_depctr instructions.
+ // Track set of SGPR writes before a given wait instruction, and search
+ // for reads of these SGPRs.
+ // Move the wait to just before the read to improve pipelining.
+ // If no related reads occur before subsequent wait then merged waits.
+ const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
+ AMDGPU::DepCtr::encodeFieldVaSdst(
+ AMDGPU::DepCtr::encodeFieldVaVcc(0, *ST), 0),
+ 0);
+ const unsigned VccLoIdx = *sgprNumber(AMDGPU::VCC_LO, *TRI);
+ const unsigned VccHiIdx = *sgprNumber(AMDGPU::VCC_HI, *TRI);
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ SmallBitVector WriteSet(128), PendingSALUWriteSet(128),
+ PendingVALUWriteSet(128);
+ MachineInstr *PrevWait = nullptr;
+
+ auto CommitWrites = [&](unsigned Mask) {
+ if (!AMDGPU::DepCtr::decodeFieldSaSdst(Mask))
+ WriteSet |= PendingSALUWriteSet;
+ bool VccLoBit = WriteSet[VccLoIdx];
+ bool VccHiBit = WriteSet[VccHiIdx];
+ if (!AMDGPU::DepCtr::decodeFieldVaSdst(Mask)) {
+ // Apply pending VALU set minus VCC bits
+ WriteSet |= PendingVALUWriteSet;
+ WriteSet[VccLoIdx] = VccLoBit;
+ WriteSet[VccHiIdx] = VccHiBit;
+ }
+ if (!AMDGPU::DepCtr::decodeFieldVaVcc(Mask)) {
+ WriteSet[VccLoIdx] = VccLoBit || PendingVALUWriteSet[VccLoIdx];
+ WriteSet[VccHiIdx] = VccHiBit || PendingVALUWriteSet[VccHiIdx];
+ }
+ // Clear all pending writes
+ PendingSALUWriteSet.reset();
+ PendingVALUWriteSet.reset();
+ };
+
+ for (MachineInstr &MI : MBB) {
+ if (MI.isMetaInstruction())
+ continue;
+
+ if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
+ (MI.getOperand(0).getImm() & ConstantMaskBits) ==
+ ConstantMaskBits) {
+ if (PrevWait) {
+ // Merge previous wait into this one.
+ MachineOperand &MaskOp = MI.getOperand(0);
+ MaskOp.setImm(
+ mergeMasks(PrevWait->getOperand(0).getImm(), MaskOp.getImm()));
+ PrevWait->eraseFromParent();
+ Changed = true;
+ } else {
+ // Starting a new region using fresh write set.
+ WriteSet.reset();
+ }
+ CommitWrites(MI.getOperand(0).getImm());
+ PrevWait = &MI;
+ continue;
+ }
+
+ // Do not optimize over branches
+ if (PrevWait && (MI.isCall() || MI.isReturn() || MI.isBranch())) {
+ PrevWait->moveBefore(&MI);
+ PrevWait = nullptr;
+ Changed = true;
+ }
+
+ const bool IsVALU = SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false);
+ const bool IsSALU = SIInstrInfo::isSALU(MI);
+ if (!IsVALU && !IsSALU)
+ continue;
+
+ for (const MachineOperand &Op : MI.operands()) {
+ if (!Op.isReg())
+ continue;
+ Register Reg = Op.getReg();
+ if (!TRI->isSGPRReg(*MRI, Reg))
+ continue;
+
+ std::optional<unsigned> RegNumber = sgprNumber(Reg, *TRI);
+ if (!RegNumber)
+ continue;
+ unsigned RegN = *RegNumber;
+ unsigned SGPRCount =
+ AMDGPU::getRegBitWidth(*TRI->getRegClassForReg(*MRI, Reg)) / 32;
+
+ if (Op.isDef()) {
+ if (IsSALU)
+ PendingSALUWriteSet.set(RegN, RegN + SGPRCount);
+ else
+ PendingVALUWriteSet.set(RegN, RegN + SGPRCount);
+ continue;
+ }
+
+ if (PrevWait &&
+ WriteSet.find_prev(RegN + SGPRCount) >= (signed)RegN) {
+ // Move the wait to here, the last point it can be valid
+ PrevWait->moveBefore(&MI);
+ PrevWait = nullptr;
+ Changed = true;
+ }
+ }
+ }
+ }
+ return Changed;
+ }
+
bool run(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
- if (!ST->hasVALUReadSGPRHazard())
+ if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
return false;
// Parse settings
@@ -464,6 +574,11 @@ class AMDGPUWaitSGPRHazards {
MRI = &MF.getRegInfo();
DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
+ // VALU mask write hazards have already been handled, but this pass
+ // performs a forward scan to optimize them.
+ if (ST->hasVALUMaskWriteHazard())
+ return ST->isWave64() ? runWaitMerging(MF) : false;
+
auto CallingConv = MF.getFunction().getCallingConv();
if (!AMDGPU::isEntryFunctionCC(CallingConv) &&
!CullSGPRHazardsOnFunctionBoundary) {
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 722ddb5d0c4dd..de7c0b42b2bf8 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -3556,17 +3556,11 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
};
SmallVector<const MachineInstr *> WaitInstrs;
- bool HasSGPRRead = false;
StateType InitialState;
// Look for SGPR write.
MachineOperand *HazardDef = nullptr;
- for (MachineOperand &Op : MI->operands()) {
- if (!Op.isReg())
- continue;
- if (Op.isDef() && HazardDef)
- continue;
-
+ for (MachineOperand &Op : MI->all_defs()) {
Register Reg = Op.getReg();
if (IgnoreableSGPR(Reg))
continue;
@@ -3576,14 +3570,9 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
if (!TRI->isSGPRReg(MRI, Reg))
continue;
}
- // Also check for SGPR reads.
- if (Op.isUse()) {
- HasSGPRRead = true;
- continue;
- }
- assert(!HazardDef);
HazardDef = &Op;
+ break;
}
if (!HazardDef)
@@ -3644,49 +3633,28 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
}
};
- const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
- AMDGPU::DepCtr::encodeFieldVaSdst(AMDGPU::DepCtr::encodeFieldVaVcc(0, ST),
- 0),
- 0);
auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
- switch (I.getOpcode()) {
- case AMDGPU::S_WAITCNT_DEPCTR:
- // Record mergable waits within region of instructions free of SGPR reads.
- if (!HasSGPRRead && I.getParent() == MI->getParent() && !I.isBundled() &&
- (I.getOperand(0).getImm() & ConstantMaskBits) == ConstantMaskBits)
- WaitInstrs.push_back(&I);
- break;
- default:
- // Update tracking of SGPR reads and writes.
- for (auto &Op : I.operands()) {
- if (!Op.isReg())
- continue;
-
- Register Reg = Op.getReg();
- if (IgnoreableSGPR(Reg))
+ // Update tracking of SGPR writes.
+ for (auto &Op : I.all_defs()) {
+ Register Reg = Op.getReg();
+ if (IgnoreableSGPR(Reg))
+ continue;
+ if (!IsVCC(Reg)) {
+ if (Op.isImplicit())
continue;
- if (!IsVCC(Reg)) {
- if (Op.isImplicit())
- continue;
- if (!TRI->isSGPRReg(MRI, Reg))
- continue;
- }
- if (Op.isUse()) {
- HasSGPRRead = true;
+ if (!TRI->isSGPRReg(MRI, Reg))
continue;
- }
+ }
- // Stop tracking any SGPRs with writes on the basis that they will
- // already have an appropriate wait inserted afterwards.
- SmallVector<Register, 2> Found;
- for (Register SGPR : State.HazardSGPRs) {
- if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
- Found.push_back(SGPR);
- }
- for (Register SGPR : Found)
- State.HazardSGPRs.erase(SGPR);
+ // Stop tracking any SGPRs with writes on the basis that they will
+ // already have an appropriate wait inserted afterwards.
+ SmallVector<Register, 2> Found;
+ for (Register SGPR : State.HazardSGPRs) {
+ if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
+ Found.push_back(SGPR);
}
- break;
+ for (Register SGPR : Found)
+ State.HazardSGPRs.erase(SGPR);
}
};
@@ -3702,39 +3670,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
: AMDGPU::DepCtr::encodeFieldVaSdst(0, ST))
: AMDGPU::DepCtr::encodeFieldSaSdst(0, ST);
- // Try to merge previous waits into this one for regions with no SGPR reads.
- if (!WaitInstrs.empty()) {
- // Note: WaitInstrs contains const pointers, so walk backward from MI to
- // obtain a mutable pointer to each instruction to be merged.
- // This is expected to be a very short walk within the same block.
- SmallVector<MachineInstr *> ToErase;
- unsigned Found = 0;
- for (MachineBasicBlock::reverse_iterator It = MI->getReverseIterator(),
- End = MI->getParent()->rend();
- Found < WaitInstrs.size() && It != End; ++It) {
- MachineInstr *WaitMI = &*It;
- // Find next wait instruction.
- if (std::as_const(WaitMI) != WaitInstrs[Found])
- continue;
- Found++;
- unsigned WaitMask = WaitMI->getOperand(0).getImm();
- assert((WaitMask & ConstantMaskBits) == ConstantMaskBits);
- DepCtr = AMDGPU::DepCtr::encodeFieldSaSdst(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldSaSdst(WaitMask),
- AMDGPU::DepCtr::decodeFieldSaSdst(DepCtr)));
- DepCtr = AMDGPU::DepCtr::encodeFieldVaSdst(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldVaSdst(WaitMask),
- AMDGPU::DepCtr::decodeFieldVaSdst(DepCtr)));
- DepCtr = AMDGPU::DepCtr::encodeFieldVaVcc(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldVaVcc(WaitMask),
- AMDGPU::DepCtr::decodeFieldVaVcc(DepCtr)));
- ToErase.push_back(WaitMI);
- }
- assert(Found == WaitInstrs.size());
- for (MachineInstr *WaitMI : ToErase)
- WaitMI->eraseFromParent();
- }
-
// Add s_waitcnt_depctr after SGPR write.
auto NextMI = std::next(MI->getIterator());
auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_dpp_lds_expected_active_lanes.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_dpp_lds_expected_active_lanes.ll
index 94509810a5ebd..be8dc995aa480 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_dpp_lds_expected_active_lanes.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_dpp_lds_expected_active_lanes.ll
@@ -331,10 +331,10 @@ define amdgpu_kernel void @add_i32_high_active_lanes(ptr addrspace(1) %out) {
; GFX1164-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-NEXT: v_readlane_b32 s6, v1, 47
; GFX1164-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
@@ -648,10 +648,10 @@ define amdgpu_kernel void @add_i32_no_metadata(ptr addrspace(1) %out) {
; GFX1164-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-NEXT: v_readlane_b32 s6, v1, 47
; GFX1164-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index ebd7f2b7b60da..8c21d42c05a2c 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -3596,44 +3596,44 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
@@ -3685,10 +3685,10 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32 v8, vcc, s2, v10
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v9, null, s3, v11, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -8743,44 +8743,44 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
@@ -8855,10 +8855,10 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v10
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e64 v9, null, s3, v11, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -15219,8 +15219,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -15276,8 +15276,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -16420,8 +16420,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
@@ -16461,7 +16461,6 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_add_f32_e32 v2, s13, v2
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX1164-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
@@ -16469,8 +16468,9 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX1164-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
@@ -16481,8 +16481,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 7a5f9c86a2326..b813e5b450e0a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1080,10 +1080,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -1625,10 +1625,10 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3074,50 +3074,49 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -3132,6 +3131,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0
@@ -3142,9 +3142,9 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -3908,29 +3908,29 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
; GFX1164_DPP-NEXT: v_permlane64_b32 v3, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_permlane64_b32 v4, v1
@@ -5196,10 +5196,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -5741,10 +5741,10 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -7221,50 +7221,49 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -7279,6 +7278,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0
@@ -7289,9 +7289,9 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB14_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -8087,10 +8087,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8941,7 +8941,6 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -8954,6 +8953,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -9725,10 +9725,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10551,7 +10551,6 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -10564,6 +10563,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -11335,10 +11335,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12161,7 +12161,6 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -12174,6 +12173,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -12945,10 +12945,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -14284,7 +14284,6 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -14299,6 +14298,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -14309,9 +14309,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB23_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -15150,10 +15150,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -16489,7 +16489,6 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -16504,6 +16503,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -16514,9 +16514,9 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB26_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -17355,10 +17355,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -18684,7 +18684,6 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -18699,6 +18698,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -18709,9 +18709,9 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB29_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -19542,10 +19542,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -20870,7 +20870,6 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -20885,6 +20884,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -20895,9 +20895,9 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB32_2
; GFX1164_DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 367d38aaff89d..4c87e6936e314 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -626,10 +626,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_or_saveexec_b64 s[10:11], -1
-; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: v_readlane_b32 s12, v1, 63
; GFX1164-NEXT: v_readlane_b32 s14, v1, 47
; GFX1164-NEXT: v_writelane_b32 v3, s13, 32
+; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: s_mov_b64 exec, s[10:11]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
index 3099aed89973c..6b3754cc039a5 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
+++ b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
@@ -42,13 +42,12 @@ body: |
; GFX11-NEXT: renamable $sgpr17 = S_CSELECT_B32 killed renamable $sgpr17, 0, implicit killed $scc
; GFX11-NEXT: renamable $vgpr7 = V_ADD_U32_e32 64, $vgpr5, implicit $exec
; GFX11-NEXT: renamable $sgpr18_sgpr19 = V_CMP_GT_U32_e64 $sgpr15, $vgpr5, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: renamable $vgpr6 = V_ADD_U32_e32 $sgpr8, $vgpr5, implicit $exec
; GFX11-NEXT: renamable $vgpr8 = V_ADD_U32_e32 $sgpr9, killed $vgpr5, implicit $exec
; GFX11-NEXT: renamable $sgpr16 = nsw S_ADD_I32 killed renamable $sgpr16, -1, implicit-def dead $scc
; GFX11-NEXT: renamable $sgpr20_sgpr21 = V_CMP_GT_U32_e64 $sgpr15, $vgpr7, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: S_CMP_LG_U32 renamable $sgpr16, 0, implicit-def $scc
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: renamable $vgpr5 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr6, $sgpr18_sgpr19, implicit $exec
; GFX11-NEXT: renamable $vgpr6 = V_ADD_U32_e32 $sgpr8, $vgpr7, implicit $exec
; GFX11-NEXT: renamable $vgpr7 = V_ADD_U32_e32 $sgpr9, killed $vgpr7, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
index 75696f6936db2..7b7a8f0ed101e 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
@@ -769,9 +769,9 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -884,9 +884,9 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -1693,9 +1693,9 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -1808,9 +1808,9 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -2617,9 +2617,9 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -2732,9 +2732,9 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -3541,9 +3541,9 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -3656,9 +3656,9 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 7cb019034dc3b..d307ffaff5c63 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -1063,14 +1063,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2290,14 +2290,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -3577,14 +3577,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4360,14 +4360,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -5672,14 +5672,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -7019,11 +7019,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -8460,11 +8460,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -9901,11 +9901,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -10824,11 +10824,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -12265,11 +12265,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index d00808bcd6836..c1e5a18ec6aa2 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -907,14 +907,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -1886,14 +1886,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2865,14 +2865,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4022,15 +4022,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5219,15 +5218,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6416,15 +6414,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 41dc73d5a1923..85bcfd88181b9 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -907,14 +907,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -1886,14 +1886,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2865,14 +2865,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4022,15 +4022,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5219,15 +5218,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6416,15 +6414,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index ccd7c7094e455..327928e152f95 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -1149,14 +1149,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2488,14 +2488,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -3827,14 +3827,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4662,14 +4662,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -6000,14 +6000,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -7347,11 +7347,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -8787,11 +8787,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -10228,11 +10228,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -11151,11 +11151,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -12591,11 +12591,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
index b20c95f9428a7..4d367c1eb3c06 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
@@ -2333,46 +2333,46 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
@@ -2380,13 +2380,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2424,46 +2424,46 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
@@ -2471,13 +2471,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index a312cfaae2d41..c876aa657338d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -2098,13 +2098,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2185,13 +2185,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
index 6215656393b17..d4c0044ce88f9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
@@ -1575,11 +1575,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1619,11 +1619,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2355,12 +2355,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2431,12 +2431,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
index 9fed78d2d8b72..31fd1123083eb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
@@ -1475,11 +1475,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1519,11 +1519,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2255,12 +2255,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2331,12 +2331,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
index 8751672583ecc..21e3c110f2fb7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
@@ -1475,11 +1475,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1519,11 +1519,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2255,12 +2255,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2331,12 +2331,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
index 43e44589ac36e..feb61672253b5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
@@ -1581,11 +1581,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1626,11 +1626,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2384,12 +2384,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2461,12 +2461,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
index 2c788270dd65d..7bc7f987bb2d4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
@@ -2197,13 +2197,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2291,13 +2291,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
index 65ce826e94d92..6f472a6e18584 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
@@ -2197,13 +2197,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2291,13 +2291,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index afee43c47f0f4..1f3d19668f86b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -2098,13 +2098,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2185,13 +2185,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
index c5e6674eea946..ae2dcfac5a4d0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
@@ -2420,51 +2420,50 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_sub_u32 s0, 0, s2
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_subb_u32 s1, 0, s3
; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2514,51 +2513,50 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_sub_u32 s0, 0, s2
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_subb_u32 s1, 0, s3
; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index ff58c56ac2f60..69b24c6f1b17a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -2201,13 +2201,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2295,13 +2295,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 8cfdac95d8de1..7a3a939891089 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -2189,13 +2189,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2283,13 +2283,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 83a98871a7287..7d6c296515733 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -2360,13 +2360,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2447,13 +2447,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index 68eef9af76aeb..d7034abfde61a 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
# RUN: llc -mtriple=amdgpu12.00 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
--- |
@@ -57,6 +57,9 @@
define amdgpu_gs void @mask_hazard_combine3() { ret void }
define amdgpu_gs void @mask_hazard_combine4() { ret void }
define amdgpu_gs void @mask_hazard_combine5() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize1() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize2() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize3() { ret void }
...
---
@@ -785,7 +788,6 @@ body: |
; GFX11-LABEL: name: mask_hazard_partial_cancel2
; GFX11: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
; GFX11-NEXT: $vcc_hi = S_MOV_B32 0
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
; GFX11-NEXT: $sgpr0 = S_MOV_B32 $vcc_lo
; GFX11-NEXT: $vcc = S_MOV_B64 1
; GFX11-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
@@ -1020,10 +1022,9 @@ body: |
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
; GFX11-NEXT: $sgpr0 = S_MOV_B32 0
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0_SaSdst_0
; GFX11-NEXT: $sgpr1 = S_MOV_B32 $sgpr4
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $vgpr5, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_SaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0_SaSdst_0
; GFX11-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: mask_hazard_combine2
@@ -1054,11 +1055,9 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: mask_hazard_combine3
@@ -1087,9 +1086,8 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: $sgpr4_sgpr5 = S_MOV_B64 $vcc
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
@@ -1123,9 +1121,8 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr1
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
@@ -1149,3 +1146,125 @@ body: |
$sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
S_ENDPGM 0
...
+
+---
+name: mask_hazard_optimize1
+body: |
+ bb.0:
+ ; GCN-LABEL: name: mask_hazard_optimize1
+ ; GCN: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GCN-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GCN-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GCN-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GCN-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GCN-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GCN-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ $sgpr4 = S_MOV_B32 $sgpr1
+ $sgpr5 = S_MOV_B32 $sgpr2
+ $sgpr6 = S_MOV_B32 $sgpr3
+ $sgpr7 = S_MOV_B32 $sgpr4
+ $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: mask_hazard_optimize2
+body: |
+ bb.0:
+ ; GFX11-LABEL: name: mask_hazard_optimize2
+ ; GFX11: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX11-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ ; GFX11-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GFX11-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: mask_hazard_optimize2
+ ; GFX12: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GFX12-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX12-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ ; GFX12-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
+ ; GFX12-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GFX12-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ $sgpr4 = S_MOV_B32 $sgpr1
+ $sgpr5 = S_MOV_B32 $sgpr2
+ $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ $sgpr6 = S_MOV_B32 $sgpr3
+ $sgpr7 = S_MOV_B32 $sgpr4
+ $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: mask_hazard_optimize3
+body: |
+ bb.0:
+ ; GFX11-LABEL: name: mask_hazard_optimize3
+ ; GFX11: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ ; GFX11-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX11-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ ; GFX11-NEXT: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ ; GFX11-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ ; GFX11-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ ; GFX11-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX11-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+ ;
+ ; GFX12-LABEL: name: mask_hazard_optimize3
+ ; GFX12: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX12-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ ; GFX12-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ ; GFX12-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ ; GFX12-NEXT: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ ; GFX12-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ ; GFX12-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ ; GFX12-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
+ ; GFX12-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+ V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ S_WAITCNT_DEPCTR 65533
+ $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+...
More information about the llvm-commits
mailing list