[llvm] Revert "[AMDGPU] Use wavefront scope for single-wave workgroup synchr… (PR #208280)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 10:59:14 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Krzysztof Drewniak (krzysz00)
<details>
<summary>Changes</summary>
This reverts commit ebc56070eb8d25c8dc73a2e97caaeb4db0f7c9fa.
Revert justified by correctness issues around DMA operations as seen in https://github.com/llvm/llvm-project/pull/207473 .
---
Patch is 305.95 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/208280.diff
8 Files Affected:
- (modified) llvm/docs/AMDGPUUsage.rst (-12)
- (modified) llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp (+4-2)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp (-4)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h (-4)
- (modified) llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp (+6-24)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll (+104-80)
- (modified) llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll (+496-176)
- (removed) llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll (-2759)
``````````diff
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 741f1f766eedd..f4bb71df30107 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -7282,18 +7282,6 @@ treated as non-atomic.
A memory synchronization scope wider than work-group is not meaningful for the
group (LDS) address space and is treated as work-group.
-When a work-group's maximum flat work-group size does not exceed the wavefront
-size, the work-group fits within a single wavefront. In this case, LLVM
-``workgroup`` synchronization scope is equivalent to ``wavefront`` scope.
-
-If the compiler can determine this bound (e.g., via ``amdgpu-flat-work-group-size``),
-the AMDGPU backend optimizes ``workgroup`` scope operations by lowering them to
-``wavefront``-scoped machine instructions.
-
-It applies to atomic ``load``, ``store``, ``atomicrmw``, and ``cmpxchg``
-instructions, and to ``fence`` instructions, when they use synchronizing memory
-orderings (``acquire``, ``release``, ``acq_rel``, or ``seq_cst``).
-
The memory model does not support the region address space which is treated as
non-atomic.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index f3fbee86168a0..59597e3dd85cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -94,8 +94,10 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(*I.getFunction());
bool IsSingleWaveWG = false;
- if (TM.getOptLevel() > CodeGenOptLevel::None)
- IsSingleWaveWG = ST.isSingleWavefrontWorkgroup(*I.getFunction());
+ if (TM.getOptLevel() > CodeGenOptLevel::None) {
+ unsigned WGMaxSize = ST.getFlatWorkGroupSizes(*I.getFunction()).second;
+ IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
+ }
IRBuilder<> B(&I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 93bf957880074..87515d22ed422 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -176,10 +176,6 @@ std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
return Requested;
}
-bool AMDGPUSubtarget::isSingleWavefrontWorkgroup(const Function &F) const {
- return getFlatWorkGroupSizes(F).second <= getWavefrontSize();
-}
-
std::pair<unsigned, unsigned> AMDGPUSubtarget::getEffectiveWavesPerEU(
std::pair<unsigned, unsigned> RequestedWavesPerEU,
std::pair<unsigned, unsigned> FlatWorkGroupSizes, unsigned LDSBytes) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index af3facc0135f0..71a8f064ffa11 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -81,10 +81,6 @@ class AMDGPUSubtarget {
/// be converted to integer, or violate subtarget's specifications.
std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
- /// \returns true if the maximum flat work-group size for \p F is at most the
- /// wavefront size, so a work-group may fit in a single wavefront.
- bool isSingleWavefrontWorkgroup(const Function &F) const;
-
/// \returns The required size of workgroups that will be used to execute \p F
/// in the \p Dim dimension, if it is known (from `!reqd_work_group_size`
/// metadata. Otherwise, returns std::nullopt.
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 409fcc7f8d980..e10898e3c1695 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -160,8 +160,7 @@ class SIMemOpInfo final {
bool IsCrossAddressSpaceOrdering = true,
AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent,
bool IsVolatile = false, bool IsNonTemporal = false,
- bool IsLastUse = false, bool IsCooperative = false,
- bool CanDemoteWorkgroupToWavefront = false, bool IsAVNone = false)
+ bool IsLastUse = false, bool IsCooperative = false, bool IsAVNone = false)
: Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope),
OrderingAddrSpace(OrderingAddrSpace), InstrAddrSpace(InstrAddrSpace),
IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
@@ -209,17 +208,6 @@ class SIMemOpInfo final {
// AGENT scope as a conservatively correct alternative.
if (this->Scope == SIAtomicScope::CLUSTER && !ST.hasClusters())
this->Scope = SIAtomicScope::AGENT;
-
- // When max flat work-group size is at most the wavefront size, the
- // work-group fits in a single wave, so LLVM workgroup scope matches
- // wavefront scope. Demote workgroup → wavefront here for fences and for
- // atomics with ordering stronger than monotonic.
- if (CanDemoteWorkgroupToWavefront &&
- this->Scope == SIAtomicScope::WORKGROUP &&
- (llvm::isStrongerThan(this->Ordering, AtomicOrdering::Monotonic) ||
- llvm::isStrongerThan(this->FailureOrdering,
- AtomicOrdering::Monotonic)))
- this->Scope = SIAtomicScope::WAVEFRONT;
}
public:
@@ -293,7 +281,6 @@ class SIMemOpAccess final {
private:
const AMDGPUMachineModuleInfo *MMI = nullptr;
const GCNSubtarget &ST;
- const bool CanDemoteWorkgroupToWavefront;
/// Reports unsupported message \p Msg for \p MI to LLVM context.
void reportUnsupported(const MachineBasicBlock::iterator &MI,
@@ -317,8 +304,7 @@ class SIMemOpAccess final {
public:
/// Construct class to support accessing the machine memory operands
/// of instructions in the machine function \p MF.
- SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST,
- const Function &F);
+ SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST);
/// \returns Load info if \p MI is a load operation, "std::nullopt" otherwise.
std::optional<SIMemOpInfo>
@@ -839,13 +825,9 @@ SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const {
return SIAtomicAddrSpace::OTHER;
}
-// TODO: Consider moving single-wave workgroup->wavefront scope relaxation to an
-// IR pass (and extending it to other scoped operations), so middle-end
-// optimizations see wavefront scope earlier.
SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_,
- const GCNSubtarget &ST, const Function &F)
- : MMI(&MMI_), ST(ST),
- CanDemoteWorkgroupToWavefront(ST.isSingleWavefrontWorkgroup(F)) {}
+ const GCNSubtarget &ST)
+ : MMI(&MMI_), ST(ST) {}
std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
const MachineBasicBlock::iterator &MI) const {
@@ -916,7 +898,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
IsNonTemporal, IsLastUse, IsCooperative,
- CanDemoteWorkgroupToWavefront, hasAVNoneMMRA(*MI));
+ hasAVNoneMMRA(*MI));
}
std::optional<SIMemOpInfo>
@@ -986,7 +968,7 @@ SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const {
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace,
SIAtomicAddrSpace::ATOMIC, IsCrossAddressSpaceOrdering,
AtomicOrdering::NotAtomic, false, false, false, false,
- CanDemoteWorkgroupToWavefront, hasAVNoneMMRA(*MI));
+ hasAVNoneMMRA(*MI));
}
std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index 678d4ffc7f094..188b830209713 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -5916,6 +5916,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn:
@@ -5924,6 +5925,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: flat_atomic_smax v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn:
@@ -5933,6 +5935,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -5945,6 +5948,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-128
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn_neg128:
@@ -5956,6 +5960,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: flat_atomic_smax v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn_neg128:
@@ -5968,6 +5973,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -5996,18 +6002,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB58_4
; GFX1250-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB58_5
; GFX1250-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB58_2
; GFX1250-SDAG-NEXT: .LBB58_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -6041,18 +6045,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB58_4
; GFX1250-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB58_5
; GFX1250-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB58_2
; GFX1250-GISEL-NEXT: .LBB58_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -6082,10 +6084,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB58_4
; GFX950-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB58_5
; GFX950-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6094,7 +6097,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
@@ -6126,10 +6128,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB58_4
; GFX950-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB58_5
; GFX950-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6138,7 +6141,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]
@@ -6180,18 +6182,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB59_4
; GFX1250-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB59_5
; GFX1250-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB59_2
; GFX1250-SDAG-NEXT: .LBB59_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -6228,18 +6228,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB59_4
; GFX1250-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB59_5
; GFX1250-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB59_2
; GFX1250-GISEL-NEXT: .LBB59_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -6272,10 +6270,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB59_4
; GFX950-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB59_5
; GFX950-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6284,7 +6283,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
@@ -6319,10 +6317,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB59_4
; GFX950-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB59_5
; GFX950-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6331,7 +6330,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]
@@ -6374,6 +6372,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/208280
More information about the llvm-commits
mailing list