[llvm] [AMDGPU] Refactor GFX11 VALU Mask Hazard Waitcnt Merging (PR #169213)
Carl Ritson via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 06:11:08 PDT 2026
https://github.com/perlfu updated https://github.com/llvm/llvm-project/pull/169213
>From 1181ef303029184dcb780137d2e3442a7982b449 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 26 Nov 2025 16:00:11 +0900
Subject: [PATCH 1/5] [AMDGPU] Refactor GFX11 VALU Mask Hazard Waitcnt Merging
Move GFX11 SGPR VALU mask hazard waitcnt merging to the a forward
scan within the AMDGPUWaitSGPRHazard pass.
This simplifies the hazard recognizer code and allows merging of
waitcnt instructions in cases where SGPRs unaffected by pending
writes are used.
In turn this greatly decreasing numbers of waits inserted in
sequences of V_CMP instructions writing SGPRs improving VALU
pipeline performance.
---
.../Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 97 +++++++++++++++++-
.../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 99 ++++---------------
.../CodeGen/AMDGPU/valu-mask-write-hazard.mir | 16 +--
3 files changed, 118 insertions(+), 94 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index 1ab11a87518eb..5d55f55fc44d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -68,6 +68,7 @@ class AMDGPUWaitSGPRHazards {
case AMDGPU::EXEC_HI:
case AMDGPU::SGPR_NULL:
case AMDGPU::SGPR_NULL64:
+ case AMDGPU::SCC:
return {};
default:
break;
@@ -442,11 +443,97 @@ class AMDGPUWaitSGPRHazards {
return Changed;
}
+ bool runWaitMerging(MachineFunction &MF) {
+ // Perform per-block merging of existing s_waitcnt_depctr instructions.
+ // Track set of SGPR writes before a given wait instruction, and search
+ // for reads of these SGPRs prior to the next wait.
+ // If no reads occur then the 1st wait can be merged into the 2nd.
+ const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
+ AMDGPU::DepCtr::encodeFieldVaSdst(
+ AMDGPU::DepCtr::encodeFieldVaVcc(0, *ST), 0),
+ 0);
+
+ bool Changed = false;
+ for (auto &MBB : MF) {
+ std::bitset<128> WriteSet, NextWriteSet;
+ MachineInstr *PrevWait = nullptr;
+ bool ReadWriteDep = false;
+ for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
+ E = MBB.instr_end();
+ MI != E; ++MI) {
+ if (MI->isMetaInstruction())
+ continue;
+
+ if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && !MI->isBundled() &&
+ (MI->getOperand(0).getImm() & ConstantMaskBits) ==
+ ConstantMaskBits) {
+ if (PrevWait && !ReadWriteDep) {
+ // Merge previous wait into this one and merge write sets.
+ MachineOperand &MaskOp = MI->getOperand(0);
+ MaskOp.setImm(
+ mergeMasks(PrevWait->getOperand(0).getImm(), MaskOp.getImm()));
+ PrevWait->eraseFromParent();
+ WriteSet |= NextWriteSet;
+ } else {
+ // Start a new merging region using fresh write set.
+ WriteSet = NextWriteSet;
+ }
+ NextWriteSet.reset();
+ PrevWait = &*MI;
+ ReadWriteDep = false;
+ Changed = true;
+ continue;
+ }
+
+ const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
+ const bool IsSALU = SIInstrInfo::isSALU(*MI);
+ if (!IsVALU && !IsSALU)
+ continue;
+
+ for (const MachineOperand &Op : MI->operands()) {
+ if (!Op.isReg())
+ continue;
+ Register Reg = Op.getReg();
+ if (!TRI->isSGPRReg(*MRI, Reg))
+ continue;
+
+ auto RegNumber = sgprNumber(Reg, *TRI);
+ if (!RegNumber)
+ continue;
+ unsigned RegN = *RegNumber;
+
+ uint8_t SGPRCount =
+ AMDGPU::getRegBitWidth(*TRI->getRegClassForReg(*MRI, Reg)) / 32;
+
+ if (Op.isDef()) {
+ for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx)
+ NextWriteSet.set(RegN + RegIdx);
+ } else {
+ if (ReadWriteDep)
+ continue;
+ for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
+ if (WriteSet[RegN + RegIdx]) {
+ ReadWriteDep = true;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ return Changed;
+ }
+
bool run(MachineFunction &MF) {
ST = &MF.getSubtarget<GCNSubtarget>();
- if (!ST->hasVALUReadSGPRHazard())
+ if (!ST->hasVALUReadSGPRHazard() && !ST->hasVALUMaskWriteHazard())
return false;
+ TII = ST->getInstrInfo();
+ TRI = ST->getRegisterInfo();
+ MRI = &MF.getRegInfo();
+ DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
+
// Parse settings
EnableSGPRHazardWaits = GlobalEnableSGPRHazardWaits;
CullSGPRHazardsOnFunctionBoundary = GlobalCullSGPRHazardsOnFunctionBoundary;
@@ -472,10 +559,10 @@ class AMDGPUWaitSGPRHazards {
if (!EnableSGPRHazardWaits)
return false;
- TII = ST->getInstrInfo();
- TRI = ST->getRegisterInfo();
- MRI = &MF.getRegInfo();
- DsNopCount = ST->isWave64() ? WAVE64_NOPS : WAVE32_NOPS;
+ // VALU mask write hazards have already been handled, but this pass
+ // performs a forward scan optimize them.
+ if (ST->hasVALUMaskWriteHazard())
+ return runWaitMerging(MF);
auto CallingConv = MF.getFunction().getCallingConv();
if (!AMDGPU::isEntryFunctionCC(CallingConv) &&
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 722ddb5d0c4dd..c06e4ec409c3b 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -3556,15 +3556,14 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
};
SmallVector<const MachineInstr *> WaitInstrs;
- bool HasSGPRRead = false;
StateType InitialState;
// Look for SGPR write.
MachineOperand *HazardDef = nullptr;
- for (MachineOperand &Op : MI->operands()) {
+ for (MachineOperand &Op : MI->all_defs()) {
if (!Op.isReg())
continue;
- if (Op.isDef() && HazardDef)
+ if (HazardDef)
continue;
Register Reg = Op.getReg();
@@ -3576,11 +3575,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
if (!TRI->isSGPRReg(MRI, Reg))
continue;
}
- // Also check for SGPR reads.
- if (Op.isUse()) {
- HasSGPRRead = true;
- continue;
- }
assert(!HazardDef);
HazardDef = &Op;
@@ -3644,49 +3638,31 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
}
};
- const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
- AMDGPU::DepCtr::encodeFieldVaSdst(AMDGPU::DepCtr::encodeFieldVaVcc(0, ST),
- 0),
- 0);
auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
- switch (I.getOpcode()) {
- case AMDGPU::S_WAITCNT_DEPCTR:
- // Record mergable waits within region of instructions free of SGPR reads.
- if (!HasSGPRRead && I.getParent() == MI->getParent() && !I.isBundled() &&
- (I.getOperand(0).getImm() & ConstantMaskBits) == ConstantMaskBits)
- WaitInstrs.push_back(&I);
- break;
- default:
- // Update tracking of SGPR reads and writes.
- for (auto &Op : I.operands()) {
- if (!Op.isReg())
- continue;
+ // Update tracking of SGPR writes.
+ for (auto &Op : I.all_defs()) {
+ if (!Op.isReg())
+ continue;
- Register Reg = Op.getReg();
- if (IgnoreableSGPR(Reg))
+ Register Reg = Op.getReg();
+ if (IgnoreableSGPR(Reg))
+ continue;
+ if (!IsVCC(Reg)) {
+ if (Op.isImplicit())
continue;
- if (!IsVCC(Reg)) {
- if (Op.isImplicit())
- continue;
- if (!TRI->isSGPRReg(MRI, Reg))
- continue;
- }
- if (Op.isUse()) {
- HasSGPRRead = true;
+ if (!TRI->isSGPRReg(MRI, Reg))
continue;
- }
+ }
- // Stop tracking any SGPRs with writes on the basis that they will
- // already have an appropriate wait inserted afterwards.
- SmallVector<Register, 2> Found;
- for (Register SGPR : State.HazardSGPRs) {
- if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
- Found.push_back(SGPR);
- }
- for (Register SGPR : Found)
- State.HazardSGPRs.erase(SGPR);
+ // Stop tracking any SGPRs with writes on the basis that they will
+ // already have an appropriate wait inserted afterwards.
+ SmallVector<Register, 2> Found;
+ for (Register SGPR : State.HazardSGPRs) {
+ if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
+ Found.push_back(SGPR);
}
- break;
+ for (Register SGPR : Found)
+ State.HazardSGPRs.erase(SGPR);
}
};
@@ -3702,39 +3678,6 @@ bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
: AMDGPU::DepCtr::encodeFieldVaSdst(0, ST))
: AMDGPU::DepCtr::encodeFieldSaSdst(0, ST);
- // Try to merge previous waits into this one for regions with no SGPR reads.
- if (!WaitInstrs.empty()) {
- // Note: WaitInstrs contains const pointers, so walk backward from MI to
- // obtain a mutable pointer to each instruction to be merged.
- // This is expected to be a very short walk within the same block.
- SmallVector<MachineInstr *> ToErase;
- unsigned Found = 0;
- for (MachineBasicBlock::reverse_iterator It = MI->getReverseIterator(),
- End = MI->getParent()->rend();
- Found < WaitInstrs.size() && It != End; ++It) {
- MachineInstr *WaitMI = &*It;
- // Find next wait instruction.
- if (std::as_const(WaitMI) != WaitInstrs[Found])
- continue;
- Found++;
- unsigned WaitMask = WaitMI->getOperand(0).getImm();
- assert((WaitMask & ConstantMaskBits) == ConstantMaskBits);
- DepCtr = AMDGPU::DepCtr::encodeFieldSaSdst(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldSaSdst(WaitMask),
- AMDGPU::DepCtr::decodeFieldSaSdst(DepCtr)));
- DepCtr = AMDGPU::DepCtr::encodeFieldVaSdst(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldVaSdst(WaitMask),
- AMDGPU::DepCtr::decodeFieldVaSdst(DepCtr)));
- DepCtr = AMDGPU::DepCtr::encodeFieldVaVcc(
- DepCtr, std::min(AMDGPU::DepCtr::decodeFieldVaVcc(WaitMask),
- AMDGPU::DepCtr::decodeFieldVaVcc(DepCtr)));
- ToErase.push_back(WaitMI);
- }
- assert(Found == WaitInstrs.size());
- for (MachineInstr *WaitMI : ToErase)
- WaitMI->eraseFromParent();
- }
-
// Add s_waitcnt_depctr after SGPR write.
auto NextMI = std::next(MI->getIterator());
auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index 8b758a75df9c3..f4d8986cb3b99 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -verify-machineinstrs -run-pass post-RA-hazard-rec,amdgpu-wait-sgpr-hazards -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
--- |
@@ -785,7 +785,6 @@ body: |
; GFX11-LABEL: name: mask_hazard_partial_cancel2
; GFX11: $vgpr1 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
; GFX11-NEXT: $vcc_hi = S_MOV_B32 0
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
; GFX11-NEXT: $sgpr0 = S_MOV_B32 $vcc_lo
; GFX11-NEXT: $vcc = S_MOV_B64 1
; GFX11-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
@@ -1020,10 +1019,9 @@ body: |
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
; GFX11-NEXT: $sgpr0 = S_MOV_B32 0
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0_SaSdst_0
; GFX11-NEXT: $sgpr1 = S_MOV_B32 $sgpr4
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $vgpr5, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_SaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0_SaSdst_0
; GFX11-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: mask_hazard_combine2
@@ -1054,11 +1052,9 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: mask_hazard_combine3
@@ -1087,9 +1083,8 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: $sgpr4_sgpr5 = S_MOV_B64 $vcc
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
@@ -1123,9 +1118,8 @@ body: |
; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr0_sgpr1, implicit $exec
; GFX11-NEXT: $vgpr5 = V_CNDMASK_B32_e64 0, $vgpr1, 0, $vgpr2, $sgpr2_sgpr3, implicit $exec
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 2, $sgpr10, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0_VaVcc_0
; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr1
; GFX11-NEXT: $sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
>From 915ecd0ae85be3cd28eff342574de22cfa85ce80 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Tue, 25 Nov 2025 21:04:07 +0900
Subject: [PATCH 2/5] - Use forward scan loop to defer waits until read
intersects write set
---
.../Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 75 ++++++++---
.../atomic_optimizations_global_pointer.ll | 68 +++++-----
.../atomic_optimizations_local_pointer.ll | 116 ++++++++--------
.../atomic_optimizations_pixelshader.ll | 2 +-
.../global-saddr-atomics-min-max-system.ll | 16 +--
.../AMDGPU/global_atomics_scan_fadd.ll | 50 +++----
.../AMDGPU/global_atomics_scan_fmax.ll | 33 +++--
.../AMDGPU/global_atomics_scan_fmin.ll | 33 +++--
.../AMDGPU/global_atomics_scan_fsub.ll | 50 +++----
.../CodeGen/AMDGPU/valu-mask-write-hazard.mir | 125 ++++++++++++++++++
10 files changed, 359 insertions(+), 209 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index 5d55f55fc44d7..44d3ff357f77c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -446,18 +446,39 @@ class AMDGPUWaitSGPRHazards {
bool runWaitMerging(MachineFunction &MF) {
// Perform per-block merging of existing s_waitcnt_depctr instructions.
// Track set of SGPR writes before a given wait instruction, and search
- // for reads of these SGPRs prior to the next wait.
- // If no reads occur then the 1st wait can be merged into the 2nd.
+ // for reads of these SGPRs.
+ // Move the wait to just before the read to improve pipelining.
+ // If no related reads occur before subsequent wait then merged waits.
const unsigned ConstantMaskBits = AMDGPU::DepCtr::encodeFieldSaSdst(
AMDGPU::DepCtr::encodeFieldVaSdst(
AMDGPU::DepCtr::encodeFieldVaVcc(0, *ST), 0),
0);
-
+ const unsigned VccLoIdx = *sgprNumber(AMDGPU::VCC_LO, *TRI);
+ const unsigned VccHiIdx = *sgprNumber(AMDGPU::VCC_HI, *TRI);
bool Changed = false;
for (auto &MBB : MF) {
- std::bitset<128> WriteSet, NextWriteSet;
+ std::bitset<128> WriteSet, PendingSALUWriteSet, PendingVALUWriteSet;
MachineInstr *PrevWait = nullptr;
- bool ReadWriteDep = false;
+
+ auto CommitWrites = [&](unsigned Mask) {
+ if (!AMDGPU::DepCtr::decodeFieldSaSdst(Mask))
+ WriteSet |= PendingSALUWriteSet;
+ bool VccLoBit = WriteSet[VccLoIdx];
+ bool VccHiBit = WriteSet[VccHiIdx];
+ if (!AMDGPU::DepCtr::decodeFieldVaSdst(Mask)) {
+ WriteSet |= PendingVALUWriteSet;
+ WriteSet.set(VccLoIdx, VccLoBit);
+ WriteSet.set(VccLoIdx, VccHiBit);
+ }
+ if (!AMDGPU::DepCtr::decodeFieldVaVcc(Mask)) {
+ WriteSet.set(VccLoIdx, VccLoBit | PendingVALUWriteSet[VccLoIdx]);
+ WriteSet.set(VccHiIdx, VccHiBit | PendingVALUWriteSet[VccHiIdx]);
+ }
+ // Clear all pending writes
+ PendingSALUWriteSet.reset();
+ PendingVALUWriteSet.reset();
+ };
+
for (MachineBasicBlock::instr_iterator MI = MBB.instr_begin(),
E = MBB.instr_end();
MI != E; ++MI) {
@@ -467,24 +488,29 @@ class AMDGPUWaitSGPRHazards {
if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && !MI->isBundled() &&
(MI->getOperand(0).getImm() & ConstantMaskBits) ==
ConstantMaskBits) {
- if (PrevWait && !ReadWriteDep) {
- // Merge previous wait into this one and merge write sets.
+ if (PrevWait) {
+ // Merge previous wait into this one.
MachineOperand &MaskOp = MI->getOperand(0);
MaskOp.setImm(
mergeMasks(PrevWait->getOperand(0).getImm(), MaskOp.getImm()));
PrevWait->eraseFromParent();
- WriteSet |= NextWriteSet;
+ Changed = true;
} else {
- // Start a new merging region using fresh write set.
- WriteSet = NextWriteSet;
+ // Starting a new region using fresh write set.
+ WriteSet.reset();
}
- NextWriteSet.reset();
+ CommitWrites(MI->getOperand(0).getImm());
PrevWait = &*MI;
- ReadWriteDep = false;
- Changed = true;
continue;
}
+ // Do not optimize over branches
+ if (PrevWait && (MI->isCall() || MI->isReturn() || MI->isBranch())) {
+ PrevWait->moveBefore(&*MI);
+ PrevWait = nullptr;
+ Changed = true;
+ }
+
const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
const bool IsSALU = SIInstrInfo::isSALU(*MI);
if (!IsVALU && !IsSALU)
@@ -506,17 +532,22 @@ class AMDGPUWaitSGPRHazards {
AMDGPU::getRegBitWidth(*TRI->getRegClassForReg(*MRI, Reg)) / 32;
if (Op.isDef()) {
- for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx)
- NextWriteSet.set(RegN + RegIdx);
- } else {
- if (ReadWriteDep)
- continue;
for (uint8_t RegIdx = 0; RegIdx < SGPRCount; ++RegIdx) {
- if (WriteSet[RegN + RegIdx]) {
- ReadWriteDep = true;
- break;
- }
+ if (IsSALU)
+ PendingSALUWriteSet.set(RegN + RegIdx);
+ else
+ PendingVALUWriteSet.set(RegN + RegIdx);
}
+ continue;
+ }
+
+ for (uint8_t RegIdx = 0; RegIdx < SGPRCount && PrevWait; ++RegIdx) {
+ if (!WriteSet[RegN + RegIdx])
+ continue;
+ // Move the wait to here, the last point it can be valid
+ PrevWait->moveBefore(&*MI);
+ PrevWait = nullptr;
+ Changed = true;
}
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 12548e219125e..91ebc656faaab 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -3596,44 +3596,44 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
@@ -3685,10 +3685,10 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32 v8, vcc, s2, v10
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v9, null, s3, v11, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -8752,44 +8752,44 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
@@ -8865,10 +8865,10 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v9
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v10
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e64 v9, null, s3, v11, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -15230,8 +15230,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -15287,8 +15287,8 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -16532,8 +16532,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
@@ -16573,7 +16573,6 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_add_f32_e32 v2, s13, v2
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX1164-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
@@ -16581,8 +16580,9 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX1164-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX1164-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
@@ -16593,8 +16593,8 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 2c6d1ee4d24b2..f1ef2cb957892 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1080,10 +1080,10 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -1625,10 +1625,10 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3074,50 +3074,49 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -3132,6 +3131,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0
@@ -3142,9 +3142,9 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -3908,29 +3908,29 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
; GFX1164_DPP-NEXT: v_permlane64_b32 v3, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_permlane64_b32 v4, v1
@@ -5196,10 +5196,10 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -5741,10 +5741,10 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -7221,50 +7221,49 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, 0
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -1
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 31
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, s3
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s2
; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -7279,6 +7278,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v8, exec_hi, v0
@@ -7289,9 +7289,9 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v7, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v8
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB14_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -8087,10 +8087,10 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -8941,7 +8941,6 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -8954,6 +8953,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -9725,10 +9725,10 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -10551,7 +10551,6 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -10564,6 +10563,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -11335,10 +11335,10 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -12161,7 +12161,6 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
@@ -12174,6 +12173,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v6, s6, 32
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -12945,10 +12945,10 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -14284,7 +14284,6 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -14299,6 +14298,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -14309,9 +14309,9 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB23_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -15150,10 +15150,10 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -16489,7 +16489,6 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -16504,6 +16503,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -16514,9 +16514,9 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB26_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -17355,10 +17355,10 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -18684,7 +18684,6 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -18699,6 +18698,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -18709,9 +18709,9 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB29_2
; GFX1164_DPP-NEXT: ; %bb.1:
@@ -19542,10 +19542,10 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 47
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s3, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
@@ -20870,7 +20870,6 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
@@ -20885,6 +20884,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s7, 32
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
@@ -20895,9 +20895,9 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s9, 48
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr7_vgpr8
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB32_2
; GFX1164_DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index f0b994420a2ca..86e97ebd7178c 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -626,10 +626,10 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_or_saveexec_b64 s[10:11], -1
-; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: v_readlane_b32 s12, v1, 63
; GFX1164-NEXT: v_readlane_b32 s14, v1, 47
; GFX1164-NEXT: v_writelane_b32 v3, s13, 32
+; GFX1164-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-NEXT: s_mov_b64 exec, s[10:11]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
index 63d63da56080b..329869f05ea5f 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll
@@ -769,9 +769,9 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -884,9 +884,9 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -1693,9 +1693,9 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -1808,9 +1808,9 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -2617,9 +2617,9 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -2732,9 +2732,9 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -3541,9 +3541,9 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
@@ -3656,9 +3656,9 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v4
; GFX11-NEXT: v_mov_b32_e32 v5, v3
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b64 exec, exec, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index ce5577e85c061..8eb602441bb47 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -1063,14 +1063,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2290,14 +2290,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -3577,14 +3577,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4360,14 +4360,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -5672,14 +5672,14 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -7019,11 +7019,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -8460,11 +8460,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -9901,11 +9901,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -10824,11 +10824,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -12265,11 +12265,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 25b079f881b98..b3696b8623598 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -907,14 +907,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -1886,14 +1886,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2865,14 +2865,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4022,15 +4022,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5219,15 +5218,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6416,15 +6414,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 5f86e306eac45..f7fb15b223c36 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -907,14 +907,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -1886,14 +1886,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2865,14 +2865,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4022,15 +4022,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5219,15 +5218,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6416,15 +6414,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 4b7e553fcddbd..cd281c30ee175 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -1149,14 +1149,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -2488,14 +2488,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -3827,14 +3827,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -4662,14 +4662,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -6000,14 +6000,14 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
@@ -7347,11 +7347,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -8787,11 +8787,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -10228,11 +10228,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -11151,11 +11151,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
@@ -12591,11 +12591,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index f4d8986cb3b99..f78b138d32e4b 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -57,6 +57,9 @@
define amdgpu_gs void @mask_hazard_combine3() { ret void }
define amdgpu_gs void @mask_hazard_combine4() { ret void }
define amdgpu_gs void @mask_hazard_combine5() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize1() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize2() { ret void }
+ define amdgpu_gs void @mask_hazard_optimize3() { ret void }
...
---
@@ -1143,3 +1146,125 @@ body: |
$sgpr2_sgpr3 = V_CMP_EQ_U32_e64 3, $sgpr10, implicit $exec
S_ENDPGM 0
...
+
+---
+name: mask_hazard_optimize1
+body: |
+ bb.0:
+ ; GCN-LABEL: name: mask_hazard_optimize1
+ ; GCN: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GCN-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GCN-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GCN-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GCN-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GCN-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GCN-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GCN-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0
+ $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ $sgpr4 = S_MOV_B32 $sgpr1
+ $sgpr5 = S_MOV_B32 $sgpr2
+ $sgpr6 = S_MOV_B32 $sgpr3
+ $sgpr7 = S_MOV_B32 $sgpr4
+ $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: mask_hazard_optimize2
+body: |
+ bb.0:
+ ; GFX11-LABEL: name: mask_hazard_optimize2
+ ; GFX11: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX11-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ ; GFX11-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GFX11-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GFX11-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: mask_hazard_optimize2
+ ; GFX12: $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
+ ; GFX12-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX12-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ ; GFX12-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR 65438
+ ; GFX12-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
+ ; GFX12-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: S_ENDPGM 0
+ $vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
+ $sgpr4 = S_MOV_B32 $sgpr1
+ $sgpr5 = S_MOV_B32 $sgpr2
+ $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
+ $sgpr6 = S_MOV_B32 $sgpr3
+ $sgpr7 = S_MOV_B32 $sgpr4
+ $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: mask_hazard_optimize3
+body: |
+ bb.0:
+ ; GFX11-LABEL: name: mask_hazard_optimize3
+ ; GFX11: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ ; GFX11-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR 65533
+ ; GFX11-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ ; GFX11-NEXT: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ ; GFX11-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ ; GFX11-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ ; GFX11-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX11-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ ; GFX11-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+ ;
+ ; GFX12-LABEL: name: mask_hazard_optimize3
+ ; GFX12: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR 65533
+ ; GFX12-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ ; GFX12-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ ; GFX12-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ ; GFX12-NEXT: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ ; GFX12-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ ; GFX12-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ ; GFX12-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX12-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ ; GFX12-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+ V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ S_WAITCNT_DEPCTR 65533
+ $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
+ $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
+ $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
+ $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
+ $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
+ $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
+ V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
+ $vgpr14 = V_ADD_U32_e32 $sgpr47, killed $vgpr14, implicit $exec
+ S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
+ $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
+ $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
+ $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
+ $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
+ $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
+...
>From d7c4924ed547d71f0fd66afe4d94c82344cdcdd3 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 26 Nov 2025 18:20:01 +0900
Subject: [PATCH 3/5] - Only need to worry about Wave64
---
llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index 44d3ff357f77c..f320cf314831c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -593,7 +593,7 @@ class AMDGPUWaitSGPRHazards {
// VALU mask write hazards have already been handled, but this pass
// performs a forward scan optimize them.
if (ST->hasVALUMaskWriteHazard())
- return runWaitMerging(MF);
+ return ST->isWave64() ? runWaitMerging(MF) : false;
auto CallingConv = MF.getFunction().getCallingConv();
if (!AMDGPU::isEntryFunctionCC(CallingConv) &&
>From 407b18fa5839304828601d5889b6602655a94079 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 1 Jul 2026 21:57:39 +0900
Subject: [PATCH 4/5] - Address reviewer comments
---
llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index f320cf314831c..9303aca7a5a49 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -456,7 +456,7 @@ class AMDGPUWaitSGPRHazards {
const unsigned VccLoIdx = *sgprNumber(AMDGPU::VCC_LO, *TRI);
const unsigned VccHiIdx = *sgprNumber(AMDGPU::VCC_HI, *TRI);
bool Changed = false;
- for (auto &MBB : MF) {
+ for (MachineBasicBlock &MBB : MF) {
std::bitset<128> WriteSet, PendingSALUWriteSet, PendingVALUWriteSet;
MachineInstr *PrevWait = nullptr;
>From bfde0ba4fba2dbfdfa87eb9c965925914bfe6594 Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson at amd.com>
Date: Wed, 1 Jul 2026 22:07:01 +0900
Subject: [PATCH 5/5] - Test updates
---
.../AMDGPU/gfx11-sgpr-hazard-latency.mir | 3 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll | 48 +++++++++----------
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll | 8 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll | 8 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll | 8 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll | 8 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll | 46 +++++++++---------
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll | 4 +-
.../CodeGen/AMDGPU/valu-mask-write-hazard.mir | 16 +++----
15 files changed, 85 insertions(+), 88 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
index 35e4122f3ff72..f969b3a30996d 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
+++ b/llvm/test/CodeGen/AMDGPU/gfx11-sgpr-hazard-latency.mir
@@ -42,13 +42,12 @@ body: |
; GFX11-NEXT: renamable $sgpr17 = S_CSELECT_B32 killed renamable $sgpr17, 0, implicit killed $scc
; GFX11-NEXT: renamable $vgpr7 = V_ADD_U32_e32 64, $vgpr5, implicit $exec
; GFX11-NEXT: renamable $sgpr18_sgpr19 = V_CMP_GT_U32_e64 $sgpr15, $vgpr5, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: renamable $vgpr6 = V_ADD_U32_e32 $sgpr8, $vgpr5, implicit $exec
; GFX11-NEXT: renamable $vgpr8 = V_ADD_U32_e32 $sgpr9, killed $vgpr5, implicit $exec
; GFX11-NEXT: renamable $sgpr16 = nsw S_ADD_I32 killed renamable $sgpr16, -1, implicit-def dead $scc
; GFX11-NEXT: renamable $sgpr20_sgpr21 = V_CMP_GT_U32_e64 $sgpr15, $vgpr7, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: S_CMP_LG_U32 renamable $sgpr16, 0, implicit-def $scc
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaSdst_0
; GFX11-NEXT: renamable $vgpr5 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr6, $sgpr18_sgpr19, implicit $exec
; GFX11-NEXT: renamable $vgpr6 = V_ADD_U32_e32 $sgpr8, $vgpr7, implicit $exec
; GFX11-NEXT: renamable $vgpr7 = V_ADD_U32_e32 $sgpr9, killed $vgpr7, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
index dbf885a1072b8..b1ba572691766 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
@@ -2069,46 +2069,46 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
@@ -2116,13 +2116,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2160,46 +2160,46 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
@@ -2207,13 +2207,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index d8d927105008d..9a229525d25d2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -1856,13 +1856,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1943,13 +1943,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
index db4743740b1ad..4a91dfcbfbca3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
@@ -1528,11 +1528,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1572,11 +1572,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2308,12 +2308,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2384,12 +2384,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
index 91daefe76a55c..6a69679ff8d99 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
@@ -1374,11 +1374,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1418,11 +1418,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2154,12 +2154,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2230,12 +2230,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
index 3ccf8d81dff6a..ae41293aefc79 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
@@ -1374,11 +1374,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1418,11 +1418,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2154,12 +2154,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2230,12 +2230,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
index 8816ff9c95ce2..026fdd6bd6d46 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
@@ -1536,11 +1536,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1581,11 +1581,11 @@ define void @divergent_value_float_dpp(ptr addrspace(1) %out, float %id.x) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
; GFX1164GISEL-NEXT: global_store_b32 v[0:1], v2, off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x2 ; 12-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v3, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:8
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2339,12 +2339,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2416,12 +2416,12 @@ define void @divergent_value_double_dpp(ptr addrspace(1) %out, double %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x3 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b64 v[7:8], off, s32 offset:20
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
index ce7f8bac0da5f..10c81693e6484 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
@@ -1943,13 +1943,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2037,13 +2037,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
index b23b3096ca9f2..a6f8c38a4f480 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
@@ -1943,13 +1943,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2037,13 +2037,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index ab122bb88316a..e6f36effc67f8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -1856,13 +1856,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1943,13 +1943,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
index bc4eab0eaa107..4be69a4d44ab1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
@@ -2150,51 +2150,50 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164DAGISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164DAGISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164DAGISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_sub_u32 s0, 0, s2
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_subb_u32 s1, 0, s3
; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s1
@@ -2244,51 +2243,50 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: v_mov_b32_e32 v7, v5
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_mov_b32_dpp v7, v7 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164GISEL-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_swizzle_b32 v7, v5 offset:swizzle(BROADCAST,32,15)
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: ds_permute_b32 v6, v8, v4
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: ds_permute_b32 v7, v8, v5
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(1)
; GFX1164GISEL-NEXT: v_add_co_u32 v4, vcc, v4, v6
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s2, v4, 63
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v5, v7, vcc
; GFX1164GISEL-NEXT: v_readlane_b32 s3, v5, 63
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_sub_u32 s0, 0, s2
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_subb_u32 s1, 0, s3
; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index b76883c6284de..d32e5089e4e1d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -1951,13 +1951,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2045,13 +2045,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index a7e9f64349218..52a7745e0af1a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -1939,13 +1939,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2033,13 +2033,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 28-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b64 v[4:5], off, s32
; GFX1164GISEL-NEXT: scratch_load_b64 v[6:7], off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32 offset:20
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:24
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 777f33c57493f..900d7fd58803c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -2094,13 +2094,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164DAGISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164DAGISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164DAGISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164DAGISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164DAGISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164DAGISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164DAGISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164DAGISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164DAGISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164DAGISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164DAGISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2181,13 +2181,13 @@ define void @divergent_value_dpp_i64(ptr addrspace(1) %out, i64 %in) #0 {
; GFX1164GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX1164GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1164GISEL-NEXT: s_xor_saveexec_b64 s[0:1], -1
-; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_clause 0x4 ; 20-byte Folded Reload
; GFX1164GISEL-NEXT: scratch_load_b32 v4, off, s32
; GFX1164GISEL-NEXT: scratch_load_b32 v5, off, s32 offset:4
; GFX1164GISEL-NEXT: scratch_load_b32 v6, off, s32 offset:8
; GFX1164GISEL-NEXT: scratch_load_b32 v7, off, s32 offset:12
; GFX1164GISEL-NEXT: scratch_load_b32 v8, off, s32 offset:16
+; GFX1164GISEL-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164GISEL-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index f78b138d32e4b..3c9faaa7f1940 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -1158,7 +1158,7 @@ body: |
; GCN-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
; GCN-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
; GCN-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
- ; GCN-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GCN-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GCN-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
; GCN-NEXT: S_ENDPGM 0
$vgpr3 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
@@ -1180,7 +1180,7 @@ body: |
; GFX11-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
; GFX11-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
; GFX11-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
- ; GFX11-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
; GFX11-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
; GFX11-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
@@ -1192,10 +1192,10 @@ body: |
; GFX12-NEXT: V_CMP_NE_U32_e32 0, $vgpr5, implicit-def $vcc, implicit $exec
; GFX12-NEXT: $sgpr4 = S_MOV_B32 $sgpr1
; GFX12-NEXT: $sgpr5 = S_MOV_B32 $sgpr2
- ; GFX12-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX12-NEXT: $sgpr4_sgpr5 = S_SWAPPC_B64 $sgpr2_sgpr3
; GFX12-NEXT: $sgpr6 = S_MOV_B32 $sgpr3
- ; GFX12-NEXT: S_WAITCNT_DEPCTR 65438
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .SaSdst_0
; GFX12-NEXT: $sgpr7 = S_MOV_B32 $sgpr4
; GFX12-NEXT: $vgpr4 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $vcc, implicit $exec
; GFX12-NEXT: S_ENDPGM 0
@@ -1219,7 +1219,7 @@ body: |
; GFX11-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
; GFX11-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
; GFX11-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
- ; GFX11-NEXT: S_WAITCNT_DEPCTR 65533
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $vgpr15 = V_CNDMASK_B32_e32 -1, killed $vgpr15, implicit $vcc, implicit $exec
; GFX11-NEXT: $vgpr16 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr16, implicit killed $vcc, implicit $exec
; GFX11-NEXT: $vgpr17 = V_ADD_U32_e32 $sgpr45, $vgpr14, implicit $exec
@@ -1228,14 +1228,14 @@ body: |
; GFX11-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
; GFX11-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
; GFX11-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
- ; GFX11-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX11-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX11-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
; GFX11-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
; GFX11-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
;
; GFX12-LABEL: name: mask_hazard_optimize3
; GFX12: V_CMP_GT_U32_e32 $sgpr8, $vgpr14, implicit-def $vcc, implicit $exec
- ; GFX12-NEXT: S_WAITCNT_DEPCTR 65533
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX12-NEXT: $vgpr16 = V_ADD_U32_e32 $sgpr47, $vgpr14, implicit $exec
; GFX12-NEXT: $vgpr14 = V_ADD_U32_e32 64, killed $vgpr14, implicit $exec
; GFX12-NEXT: $sgpr56 = nsw S_ADD_I32 killed $sgpr56, -1, implicit-def dead $scc
@@ -1247,7 +1247,7 @@ body: |
; GFX12-NEXT: S_CMP_LG_U32 $sgpr56, 0, implicit-def $scc
; GFX12-NEXT: $vgpr18 = V_ADD_U32_e32 $sgpr52, $vgpr16, implicit $exec
; GFX12-NEXT: $vgpr19 = V_ADD_U32_e32 $sgpr53, $vgpr16, implicit $exec
- ; GFX12-NEXT: S_WAITCNT_DEPCTR 65437
+ ; GFX12-NEXT: S_WAITCNT_DEPCTR .VaVcc_0
; GFX12-NEXT: $vgpr17 = V_CNDMASK_B32_e32 -1, killed $vgpr17, implicit $vcc, implicit $exec
; GFX12-NEXT: $vgpr14 = V_CNDMASK_B32_e32 $sgpr49, killed $vgpr14, implicit killed $vcc, implicit $exec
; GFX12-NEXT: $vgpr20 = V_ADD_U32_e32 $sgpr54, $vgpr16, implicit $exec
More information about the llvm-commits
mailing list