[llvm] Revert "[AMDGPU] Use wavefront scope for single-wave workgroup synchr… (PR #208280)
Krzysztof Drewniak via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 10:58:37 PDT 2026
https://github.com/krzysz00 created https://github.com/llvm/llvm-project/pull/208280
This reverts commit ebc56070eb8d25c8dc73a2e97caaeb4db0f7c9fa.
Revert justified by correctness issues around DMA operations as seen in https://github.com/llvm/llvm-project/pull/207473 .
>From 4d5dfe2ae0752a5cdae3088d4a73579d4e71388b Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak at amd.com>
Date: Wed, 8 Jul 2026 17:53:50 +0000
Subject: [PATCH] Revert "[AMDGPU] Use wavefront scope for single-wave
workgroup synchronization (#187673)"
This reverts commit ebc56070eb8d25c8dc73a2e97caaeb4db0f7c9fa.
Revert justified by correctness issues around DMA operations as seen
in https://github.com/llvm/llvm-project/pull/207473 .
---
llvm/docs/AMDGPUUsage.rst | 12 -
.../Target/AMDGPU/AMDGPULowerIntrinsics.cpp | 6 +-
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 4 -
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h | 4 -
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 30 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 184 +-
.../CodeGen/AMDGPU/global-saddr-atomics.ll | 672 ++--
...-legalizer-single-wave-workgroup-memops.ll | 2759 -----------------
8 files changed, 610 insertions(+), 3061 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 741f1f766eedd..f4bb71df30107 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -7282,18 +7282,6 @@ treated as non-atomic.
A memory synchronization scope wider than work-group is not meaningful for the
group (LDS) address space and is treated as work-group.
-When a work-group's maximum flat work-group size does not exceed the wavefront
-size, the work-group fits within a single wavefront. In this case, LLVM
-``workgroup`` synchronization scope is equivalent to ``wavefront`` scope.
-
-If the compiler can determine this bound (e.g., via ``amdgpu-flat-work-group-size``),
-the AMDGPU backend optimizes ``workgroup`` scope operations by lowering them to
-``wavefront``-scoped machine instructions.
-
-It applies to atomic ``load``, ``store``, ``atomicrmw``, and ``cmpxchg``
-instructions, and to ``fence`` instructions, when they use synchronizing memory
-orderings (``acquire``, ``release``, ``acq_rel``, or ``seq_cst``).
-
The memory model does not support the region address space which is treated as
non-atomic.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index f3fbee86168a0..59597e3dd85cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -94,8 +94,10 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(*I.getFunction());
bool IsSingleWaveWG = false;
- if (TM.getOptLevel() > CodeGenOptLevel::None)
- IsSingleWaveWG = ST.isSingleWavefrontWorkgroup(*I.getFunction());
+ if (TM.getOptLevel() > CodeGenOptLevel::None) {
+ unsigned WGMaxSize = ST.getFlatWorkGroupSizes(*I.getFunction()).second;
+ IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
+ }
IRBuilder<> B(&I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 93bf957880074..87515d22ed422 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -176,10 +176,6 @@ std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
return Requested;
}
-bool AMDGPUSubtarget::isSingleWavefrontWorkgroup(const Function &F) const {
- return getFlatWorkGroupSizes(F).second <= getWavefrontSize();
-}
-
std::pair<unsigned, unsigned> AMDGPUSubtarget::getEffectiveWavesPerEU(
std::pair<unsigned, unsigned> RequestedWavesPerEU,
std::pair<unsigned, unsigned> FlatWorkGroupSizes, unsigned LDSBytes) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index af3facc0135f0..71a8f064ffa11 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -81,10 +81,6 @@ class AMDGPUSubtarget {
/// be converted to integer, or violate subtarget's specifications.
std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
- /// \returns true if the maximum flat work-group size for \p F is at most the
- /// wavefront size, so a work-group may fit in a single wavefront.
- bool isSingleWavefrontWorkgroup(const Function &F) const;
-
/// \returns The required size of workgroups that will be used to execute \p F
/// in the \p Dim dimension, if it is known (from `!reqd_work_group_size`
/// metadata. Otherwise, returns std::nullopt.
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 409fcc7f8d980..e10898e3c1695 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -160,8 +160,7 @@ class SIMemOpInfo final {
bool IsCrossAddressSpaceOrdering = true,
AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent,
bool IsVolatile = false, bool IsNonTemporal = false,
- bool IsLastUse = false, bool IsCooperative = false,
- bool CanDemoteWorkgroupToWavefront = false, bool IsAVNone = false)
+ bool IsLastUse = false, bool IsCooperative = false, bool IsAVNone = false)
: Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope),
OrderingAddrSpace(OrderingAddrSpace), InstrAddrSpace(InstrAddrSpace),
IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
@@ -209,17 +208,6 @@ class SIMemOpInfo final {
// AGENT scope as a conservatively correct alternative.
if (this->Scope == SIAtomicScope::CLUSTER && !ST.hasClusters())
this->Scope = SIAtomicScope::AGENT;
-
- // When max flat work-group size is at most the wavefront size, the
- // work-group fits in a single wave, so LLVM workgroup scope matches
- // wavefront scope. Demote workgroup → wavefront here for fences and for
- // atomics with ordering stronger than monotonic.
- if (CanDemoteWorkgroupToWavefront &&
- this->Scope == SIAtomicScope::WORKGROUP &&
- (llvm::isStrongerThan(this->Ordering, AtomicOrdering::Monotonic) ||
- llvm::isStrongerThan(this->FailureOrdering,
- AtomicOrdering::Monotonic)))
- this->Scope = SIAtomicScope::WAVEFRONT;
}
public:
@@ -293,7 +281,6 @@ class SIMemOpAccess final {
private:
const AMDGPUMachineModuleInfo *MMI = nullptr;
const GCNSubtarget &ST;
- const bool CanDemoteWorkgroupToWavefront;
/// Reports unsupported message \p Msg for \p MI to LLVM context.
void reportUnsupported(const MachineBasicBlock::iterator &MI,
@@ -317,8 +304,7 @@ class SIMemOpAccess final {
public:
/// Construct class to support accessing the machine memory operands
/// of instructions in the machine function \p MF.
- SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST,
- const Function &F);
+ SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST);
/// \returns Load info if \p MI is a load operation, "std::nullopt" otherwise.
std::optional<SIMemOpInfo>
@@ -839,13 +825,9 @@ SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const {
return SIAtomicAddrSpace::OTHER;
}
-// TODO: Consider moving single-wave workgroup->wavefront scope relaxation to an
-// IR pass (and extending it to other scoped operations), so middle-end
-// optimizations see wavefront scope earlier.
SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_,
- const GCNSubtarget &ST, const Function &F)
- : MMI(&MMI_), ST(ST),
- CanDemoteWorkgroupToWavefront(ST.isSingleWavefrontWorkgroup(F)) {}
+ const GCNSubtarget &ST)
+ : MMI(&MMI_), ST(ST) {}
std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
const MachineBasicBlock::iterator &MI) const {
@@ -916,7 +898,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
IsNonTemporal, IsLastUse, IsCooperative,
- CanDemoteWorkgroupToWavefront, hasAVNoneMMRA(*MI));
+ hasAVNoneMMRA(*MI));
}
std::optional<SIMemOpInfo>
@@ -986,7 +968,7 @@ SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const {
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace,
SIAtomicAddrSpace::ATOMIC, IsCrossAddressSpaceOrdering,
AtomicOrdering::NotAtomic, false, false, false, false,
- CanDemoteWorkgroupToWavefront, hasAVNoneMMRA(*MI));
+ hasAVNoneMMRA(*MI));
}
std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index 678d4ffc7f094..188b830209713 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -5916,6 +5916,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn:
@@ -5924,6 +5925,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: flat_atomic_smax v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn:
@@ -5933,6 +5935,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -5945,6 +5948,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-128
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn_neg128:
@@ -5956,6 +5960,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: flat_atomic_smax v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn_neg128:
@@ -5968,6 +5973,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -5996,18 +6002,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB58_4
; GFX1250-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB58_5
; GFX1250-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB58_2
; GFX1250-SDAG-NEXT: .LBB58_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -6041,18 +6045,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB58_4
; GFX1250-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB58_5
; GFX1250-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB58_2
; GFX1250-GISEL-NEXT: .LBB58_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -6082,10 +6084,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB58_4
; GFX950-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB58_5
; GFX950-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6094,7 +6097,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
@@ -6126,10 +6128,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB58_4
; GFX950-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB58_5
; GFX950-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6138,7 +6141,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]
@@ -6180,18 +6182,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB59_4
; GFX1250-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB59_5
; GFX1250-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB59_2
; GFX1250-SDAG-NEXT: .LBB59_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -6228,18 +6228,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB59_4
; GFX1250-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB59_5
; GFX1250-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB59_2
; GFX1250-GISEL-NEXT: .LBB59_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -6272,10 +6270,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB59_4
; GFX950-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB59_5
; GFX950-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6284,7 +6283,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
@@ -6319,10 +6317,11 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB59_4
; GFX950-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB59_5
; GFX950-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6331,7 +6330,6 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]
@@ -6374,6 +6372,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB60_2
@@ -6411,6 +6410,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB60_2
@@ -6443,6 +6443,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB60_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6479,6 +6480,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB60_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6526,6 +6528,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB61_2
@@ -6566,6 +6569,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3] offset:-128
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB61_2
@@ -6601,6 +6605,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB61_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6641,6 +6646,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB61_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6746,6 +6752,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_min_saddr_i32_nortn:
@@ -6754,6 +6761,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: flat_atomic_smin v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_min_saddr_i32_nortn:
@@ -6763,6 +6771,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smin v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -6775,6 +6784,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] offset:-128
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_min_saddr_i32_nortn_neg128:
@@ -6786,6 +6796,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: flat_atomic_smin v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_min_saddr_i32_nortn_neg128:
@@ -6798,6 +6809,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_smin v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -6826,18 +6838,16 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB66_4
; GFX1250-SDAG-NEXT: .LBB66_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB66_5
; GFX1250-SDAG-NEXT: .LBB66_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB66_2
; GFX1250-SDAG-NEXT: .LBB66_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -6871,18 +6881,16 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB66_4
; GFX1250-GISEL-NEXT: .LBB66_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB66_5
; GFX1250-GISEL-NEXT: .LBB66_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB66_2
; GFX1250-GISEL-NEXT: .LBB66_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -6912,10 +6920,11 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB66_4
; GFX950-SDAG-NEXT: .LBB66_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB66_5
; GFX950-SDAG-NEXT: .LBB66_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6924,7 +6933,6 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
@@ -6956,10 +6964,11 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB66_4
; GFX950-GISEL-NEXT: .LBB66_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB66_5
; GFX950-GISEL-NEXT: .LBB66_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -6968,7 +6977,6 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]
@@ -7010,18 +7018,16 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB67_4
; GFX1250-SDAG-NEXT: .LBB67_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB67_5
; GFX1250-SDAG-NEXT: .LBB67_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB67_2
; GFX1250-SDAG-NEXT: .LBB67_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -7058,18 +7064,16 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB67_4
; GFX1250-GISEL-NEXT: .LBB67_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB67_5
; GFX1250-GISEL-NEXT: .LBB67_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB67_2
; GFX1250-GISEL-NEXT: .LBB67_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -7102,10 +7106,11 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB67_4
; GFX950-SDAG-NEXT: .LBB67_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB67_5
; GFX950-SDAG-NEXT: .LBB67_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7114,7 +7119,6 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
@@ -7149,10 +7153,11 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB67_4
; GFX950-GISEL-NEXT: .LBB67_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB67_5
; GFX950-GISEL-NEXT: .LBB67_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7161,7 +7166,6 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]
@@ -7204,6 +7208,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB68_2
@@ -7241,6 +7246,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB68_2
@@ -7273,6 +7279,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB68_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7309,6 +7316,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB68_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7356,6 +7364,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB69_2
@@ -7396,6 +7405,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3] offset:-128
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB69_2
@@ -7431,6 +7441,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB69_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7471,6 +7482,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB69_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7576,6 +7588,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umax_saddr_i32_nortn:
@@ -7584,6 +7597,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: flat_atomic_umax v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i32_nortn:
@@ -7593,6 +7607,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umax v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -7605,6 +7620,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] offset:-128
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umax_saddr_i32_nortn_neg128:
@@ -7616,6 +7632,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: flat_atomic_umax v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_umax_saddr_i32_nortn_neg128:
@@ -7628,6 +7645,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umax v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -7656,18 +7674,16 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB74_4
; GFX1250-SDAG-NEXT: .LBB74_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB74_5
; GFX1250-SDAG-NEXT: .LBB74_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB74_2
; GFX1250-SDAG-NEXT: .LBB74_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -7701,18 +7717,16 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB74_4
; GFX1250-GISEL-NEXT: .LBB74_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB74_5
; GFX1250-GISEL-NEXT: .LBB74_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB74_2
; GFX1250-GISEL-NEXT: .LBB74_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -7742,10 +7756,11 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB74_4
; GFX950-SDAG-NEXT: .LBB74_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB74_5
; GFX950-SDAG-NEXT: .LBB74_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7754,7 +7769,6 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
@@ -7786,10 +7800,11 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB74_4
; GFX950-GISEL-NEXT: .LBB74_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB74_5
; GFX950-GISEL-NEXT: .LBB74_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7798,7 +7813,6 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5]
@@ -7840,18 +7854,16 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB75_4
; GFX1250-SDAG-NEXT: .LBB75_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB75_5
; GFX1250-SDAG-NEXT: .LBB75_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB75_2
; GFX1250-SDAG-NEXT: .LBB75_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -7888,18 +7900,16 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB75_4
; GFX1250-GISEL-NEXT: .LBB75_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB75_5
; GFX1250-GISEL-NEXT: .LBB75_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB75_2
; GFX1250-GISEL-NEXT: .LBB75_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -7932,10 +7942,11 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB75_4
; GFX950-SDAG-NEXT: .LBB75_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB75_5
; GFX950-SDAG-NEXT: .LBB75_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7944,7 +7955,6 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
@@ -7979,10 +7989,11 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB75_4
; GFX950-GISEL-NEXT: .LBB75_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB75_5
; GFX950-GISEL-NEXT: .LBB75_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -7991,7 +8002,6 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5]
@@ -8034,6 +8044,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB76_2
@@ -8071,6 +8082,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB76_2
@@ -8103,6 +8115,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB76_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8139,6 +8152,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB76_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8186,6 +8200,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB77_2
@@ -8226,6 +8241,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3] offset:-128
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB77_2
@@ -8261,6 +8277,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB77_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8301,6 +8318,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB77_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8406,6 +8424,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3]
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umin_saddr_i32_nortn:
@@ -8414,6 +8433,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]
; GFX950-SDAG-NEXT: flat_atomic_umin v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i32_nortn:
@@ -8423,6 +8443,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umin v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -8435,6 +8456,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] offset:-128
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umin_saddr_i32_nortn_neg128:
@@ -8446,6 +8468,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
; GFX950-SDAG-NEXT: flat_atomic_umin v[0:1], v2
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: flat_umin_saddr_i32_nortn_neg128:
@@ -8458,6 +8481,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc
; GFX950-GISEL-NEXT: flat_atomic_umin v[2:3], v1
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -8486,18 +8510,16 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB82_4
; GFX1250-SDAG-NEXT: .LBB82_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB82_5
; GFX1250-SDAG-NEXT: .LBB82_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB82_2
; GFX1250-SDAG-NEXT: .LBB82_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -8531,18 +8553,16 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB82_4
; GFX1250-GISEL-NEXT: .LBB82_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB82_5
; GFX1250-GISEL-NEXT: .LBB82_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB82_2
; GFX1250-GISEL-NEXT: .LBB82_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -8572,10 +8592,11 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB82_4
; GFX950-SDAG-NEXT: .LBB82_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB82_5
; GFX950-SDAG-NEXT: .LBB82_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8584,7 +8605,6 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
@@ -8616,10 +8636,11 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB82_4
; GFX950-GISEL-NEXT: .LBB82_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB82_5
; GFX950-GISEL-NEXT: .LBB82_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8628,7 +8649,6 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
@@ -8670,18 +8690,16 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB83_4
; GFX1250-SDAG-NEXT: .LBB83_2: ; %atomicrmw.phi
; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_branch .LBB83_5
; GFX1250-SDAG-NEXT: .LBB83_3: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB83_2
; GFX1250-SDAG-NEXT: .LBB83_4: ; %atomicrmw.private
; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
@@ -8718,18 +8736,16 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB83_4
; GFX1250-GISEL-NEXT: .LBB83_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB83_5
; GFX1250-GISEL-NEXT: .LBB83_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB83_2
; GFX1250-GISEL-NEXT: .LBB83_4: ; %atomicrmw.private
; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
@@ -8762,10 +8778,11 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB83_4
; GFX950-SDAG-NEXT: .LBB83_2: ; %atomicrmw.phi
; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: s_branch .LBB83_5
; GFX950-SDAG-NEXT: .LBB83_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] sc0
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8774,7 +8791,6 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
-; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX950-SDAG-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
@@ -8809,10 +8825,11 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB83_4
; GFX950-GISEL-NEXT: .LBB83_2: ; %atomicrmw.phi
; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: s_branch .LBB83_5
; GFX950-GISEL-NEXT: .LBB83_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[4:5] sc0
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8821,7 +8838,6 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc
-; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX950-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
@@ -8864,6 +8880,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB84_2
@@ -8901,6 +8918,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB84_2
@@ -8933,6 +8951,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB84_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -8969,6 +8988,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB84_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -9016,6 +9036,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB85_2
@@ -9056,6 +9077,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3] offset:-128
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB85_2
@@ -9091,6 +9113,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-SDAG-NEXT: s_endpgm
; GFX950-SDAG-NEXT: .LBB85_3: ; %atomicrmw.global
; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]
+; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
@@ -9131,6 +9154,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX950-GISEL-NEXT: s_endpgm
; GFX950-GISEL-NEXT: .LBB85_3: ; %atomicrmw.global
; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5]
+; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll
index 466d7152a0d84..58f7c4340276d 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll
@@ -2142,22 +2142,31 @@ define amdgpu_ps void @global_xor_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; --------------------------------------------------------------------------------
define amdgpu_ps float @global_max_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_max_saddr_i32_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax v0, v0, v1, s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_max_saddr_i32_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax v0, v0, v1, s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_max_saddr_i32_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax v0, v0, v1, s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_max_saddr_i32_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i32 v0, v0, v1, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_max_saddr_i32_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2167,22 +2176,31 @@ define amdgpu_ps float @global_max_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps float @global_max_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_max_saddr_i32_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax v0, v0, v1, s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_max_saddr_i32_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax v0, v0, v1, s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_max_saddr_i32_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax v0, v0, v1, s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_max_saddr_i32_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_max_saddr_i32_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2193,19 +2211,30 @@ define amdgpu_ps float @global_max_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
}
define amdgpu_ps void @global_max_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_max_saddr_i32_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax v0, v1, s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_max_saddr_i32_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax v0, v1, s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_max_saddr_i32_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax v0, v1, s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_max_saddr_i32_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i32 v0, v1, s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_max_saddr_i32_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i32 v0, v1, s[2:3]
+; GFX12-NEXT: global_atomic_max_i32 v0, v1, s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2214,19 +2243,30 @@ define amdgpu_ps void @global_max_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps void @global_max_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_max_saddr_i32_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax v0, v1, s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_max_saddr_i32_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax v0, v1, s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_max_saddr_i32_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax v0, v1, s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_max_saddr_i32_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i32 v0, v1, s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_max_saddr_i32_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i32 v0, v1, s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_max_i32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2236,22 +2276,31 @@ define amdgpu_ps void @global_max_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
}
define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_max_saddr_i64_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax_x2 v[0:1], v0, v[1:2], s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_max_saddr_i64_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_max_saddr_i64_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_max_saddr_i64_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i64 v[0:1], v0, v[1:2], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_max_saddr_i64_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_i64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2261,22 +2310,31 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn(ptr addrspace(1) inreg %s
}
define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_max_saddr_i64_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_max_saddr_i64_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_max_saddr_i64_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_max_saddr_i64_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_max_saddr_i64_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_i64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2287,19 +2345,30 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn_neg128(ptr addrspace(1) i
}
define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_max_saddr_i64_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax_x2 v0, v[1:2], s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_max_saddr_i64_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax_x2 v0, v[1:2], s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_max_saddr_i64_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax_x2 v0, v[1:2], s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_max_saddr_i64_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i64 v0, v[1:2], s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_max_saddr_i64_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i64 v0, v[1:2], s[2:3]
+; GFX12-NEXT: global_atomic_max_i64 v0, v[1:2], s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2308,19 +2377,30 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_max_saddr_i64_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smax_x2 v0, v[1:2], s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_max_saddr_i64_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smax_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_max_saddr_i64_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smax_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_max_saddr_i64_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_i64 v0, v[1:2], s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_max_saddr_i64_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_i64 v0, v[1:2], s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_max_i64 v0, v[1:2], s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2334,22 +2414,31 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; --------------------------------------------------------------------------------
define amdgpu_ps float @global_min_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_min_saddr_i32_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin v0, v0, v1, s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_min_saddr_i32_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin v0, v0, v1, s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_min_saddr_i32_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin v0, v0, v1, s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_min_saddr_i32_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i32 v0, v0, v1, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_min_saddr_i32_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2359,22 +2448,31 @@ define amdgpu_ps float @global_min_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps float @global_min_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_min_saddr_i32_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin v0, v0, v1, s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_min_saddr_i32_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin v0, v0, v1, s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_min_saddr_i32_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin v0, v0, v1, s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_min_saddr_i32_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_min_saddr_i32_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2385,19 +2483,30 @@ define amdgpu_ps float @global_min_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
}
define amdgpu_ps void @global_min_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_min_saddr_i32_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin v0, v1, s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_min_saddr_i32_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin v0, v1, s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_min_saddr_i32_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin v0, v1, s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_min_saddr_i32_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i32 v0, v1, s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_min_saddr_i32_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i32 v0, v1, s[2:3]
+; GFX12-NEXT: global_atomic_min_i32 v0, v1, s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2406,19 +2515,30 @@ define amdgpu_ps void @global_min_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps void @global_min_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_min_saddr_i32_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin v0, v1, s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_min_saddr_i32_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin v0, v1, s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_min_saddr_i32_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin v0, v1, s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_min_saddr_i32_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i32 v0, v1, s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_min_saddr_i32_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i32 v0, v1, s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_min_i32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2428,22 +2548,31 @@ define amdgpu_ps void @global_min_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
}
define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_min_saddr_i64_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin_x2 v[0:1], v0, v[1:2], s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_min_saddr_i64_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_min_saddr_i64_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_min_saddr_i64_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i64 v[0:1], v0, v[1:2], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_min_saddr_i64_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_i64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2453,22 +2582,31 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn(ptr addrspace(1) inreg %s
}
define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_min_saddr_i64_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_min_saddr_i64_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_min_saddr_i64_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_min_saddr_i64_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_min_saddr_i64_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_i64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2479,19 +2617,30 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn_neg128(ptr addrspace(1) i
}
define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_min_saddr_i64_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin_x2 v0, v[1:2], s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_min_saddr_i64_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin_x2 v0, v[1:2], s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_min_saddr_i64_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin_x2 v0, v[1:2], s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_min_saddr_i64_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i64 v0, v[1:2], s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_min_saddr_i64_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i64 v0, v[1:2], s[2:3]
+; GFX12-NEXT: global_atomic_min_i64 v0, v[1:2], s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2500,19 +2649,30 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_min_saddr_i64_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_smin_x2 v0, v[1:2], s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_min_saddr_i64_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_smin_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_min_saddr_i64_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_smin_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_min_saddr_i64_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_i64 v0, v[1:2], s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_min_saddr_i64_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_i64 v0, v[1:2], s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_min_i64 v0, v[1:2], s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2526,22 +2686,31 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; --------------------------------------------------------------------------------
define amdgpu_ps float @global_umax_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umax_saddr_i32_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax v0, v0, v1, s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umax_saddr_i32_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax v0, v0, v1, s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umax_saddr_i32_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax v0, v0, v1, s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umax_saddr_i32_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u32 v0, v0, v1, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umax_saddr_i32_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2551,22 +2720,31 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps float @global_umax_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umax_saddr_i32_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax v0, v0, v1, s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umax_saddr_i32_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax v0, v0, v1, s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umax_saddr_i32_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax v0, v0, v1, s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umax_saddr_i32_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umax_saddr_i32_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2577,19 +2755,30 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn_neg128(ptr addrspace(1) inreg
}
define amdgpu_ps void @global_umax_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umax_saddr_i32_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax v0, v1, s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umax_saddr_i32_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax v0, v1, s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umax_saddr_i32_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax v0, v1, s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umax_saddr_i32_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u32 v0, v1, s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umax_saddr_i32_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u32 v0, v1, s[2:3]
+; GFX12-NEXT: global_atomic_max_u32 v0, v1, s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2598,19 +2787,30 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn(ptr addrspace(1) inreg %sbase
}
define amdgpu_ps void @global_umax_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umax_saddr_i32_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax v0, v1, s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umax_saddr_i32_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax v0, v1, s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umax_saddr_i32_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax v0, v1, s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umax_saddr_i32_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u32 v0, v1, s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umax_saddr_i32_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u32 v0, v1, s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_max_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2620,22 +2820,31 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
}
define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umax_saddr_i64_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax_x2 v[0:1], v0, v[1:2], s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umax_saddr_i64_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umax_saddr_i64_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umax_saddr_i64_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u64 v[0:1], v0, v[1:2], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umax_saddr_i64_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_u64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2645,22 +2854,31 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn(ptr addrspace(1) inreg %
}
define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umax_saddr_i64_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umax_saddr_i64_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umax_saddr_i64_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umax_saddr_i64_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umax_saddr_i64_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_max_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2671,19 +2889,30 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn_neg128(ptr addrspace(1)
}
define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umax_saddr_i64_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax_x2 v0, v[1:2], s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umax_saddr_i64_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax_x2 v0, v[1:2], s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umax_saddr_i64_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax_x2 v0, v[1:2], s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umax_saddr_i64_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u64 v0, v[1:2], s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umax_saddr_i64_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u64 v0, v[1:2], s[2:3]
+; GFX12-NEXT: global_atomic_max_u64 v0, v[1:2], s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2692,19 +2921,30 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
}
define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umax_saddr_i64_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umax_x2 v0, v[1:2], s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umax_saddr_i64_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umax_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umax_saddr_i64_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umax_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umax_saddr_i64_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_max_u64 v0, v[1:2], s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umax_saddr_i64_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_max_u64 v0, v[1:2], s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_max_u64 v0, v[1:2], s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2718,22 +2958,31 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
; --------------------------------------------------------------------------------
define amdgpu_ps float @global_umin_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umin_saddr_i32_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin v0, v0, v1, s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umin_saddr_i32_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin v0, v0, v1, s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umin_saddr_i32_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin v0, v0, v1, s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umin_saddr_i32_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u32 v0, v0, v1, s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umin_saddr_i32_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2743,22 +2992,31 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
}
define amdgpu_ps float @global_umin_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umin_saddr_i32_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin v0, v0, v1, s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umin_saddr_i32_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin v0, v0, v1, s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umin_saddr_i32_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin v0, v0, v1, s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umin_saddr_i32_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umin_saddr_i32_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2769,19 +3027,30 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn_neg128(ptr addrspace(1) inreg
}
define amdgpu_ps void @global_umin_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umin_saddr_i32_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin v0, v1, s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umin_saddr_i32_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin v0, v1, s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umin_saddr_i32_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin v0, v1, s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umin_saddr_i32_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u32 v0, v1, s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umin_saddr_i32_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u32 v0, v1, s[2:3]
+; GFX12-NEXT: global_atomic_min_u32 v0, v1, s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2790,19 +3059,30 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn(ptr addrspace(1) inreg %sbase
}
define amdgpu_ps void @global_umin_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i32 %data) {
-; GCN-LABEL: global_umin_saddr_i32_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin v0, v1, s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umin_saddr_i32_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin v0, v1, s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umin_saddr_i32_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin v0, v1, s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umin_saddr_i32_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u32 v0, v1, s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umin_saddr_i32_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u32 v0, v1, s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_min_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2812,22 +3092,31 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
}
define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umin_saddr_i64_rtn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin_x2 v[0:1], v0, v[1:2], s[2:3] glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umin_saddr_i64_rtn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umin_saddr_i64_rtn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin_x2 v[0:1], v0, v[1:2], s[2:3] glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umin_saddr_i64_rtn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u64 v[0:1], v0, v[1:2], s[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umin_saddr_i64_rtn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_u64 v[0:1], v0, v[1:2], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2837,22 +3126,31 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn(ptr addrspace(1) inreg %
}
define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umin_saddr_i64_rtn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; return to shader part epilog
+; GFX9-LABEL: global_umin_saddr_i64_rtn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: ; return to shader part epilog
+;
+; GFX10-LABEL: global_umin_saddr_i64_rtn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: global_umin_saddr_i64_rtn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: global_umin_saddr_i64_rtn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: global_atomic_min_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2863,19 +3161,30 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn_neg128(ptr addrspace(1)
}
define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umin_saddr_i64_nortn:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin_x2 v0, v[1:2], s[2:3]
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umin_saddr_i64_nortn:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin_x2 v0, v[1:2], s[2:3]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umin_saddr_i64_nortn:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin_x2 v0, v[1:2], s[2:3]
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umin_saddr_i64_nortn:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u64 v0, v[1:2], s[2:3]
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umin_saddr_i64_nortn:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u64 v0, v[1:2], s[2:3]
+; GFX12-NEXT: global_atomic_min_u64 v0, v[1:2], s[2:3] scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -2884,19 +3193,30 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
}
define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %sbase, i32 %voffset, i64 %data) {
-; GCN-LABEL: global_umin_saddr_i64_nortn_neg128:
-; GCN: ; %bb.0:
-; GCN-NEXT: global_atomic_umin_x2 v0, v[1:2], s[2:3] offset:-128
-; GCN-NEXT: s_endpgm
+; GFX9-LABEL: global_umin_saddr_i64_nortn_neg128:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_atomic_umin_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: global_umin_saddr_i64_nortn_neg128:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_atomic_umin_x2 v0, v[1:2], s[2:3] offset:-128
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_umin_saddr_i64_nortn_neg128:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_atomic_min_u64 v0, v[1:2], s[2:3] offset:-128
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: global_umin_saddr_i64_nortn_neg128:
; GFX12: ; %bb.0:
-; GFX12-NEXT: global_atomic_min_u64 v0, v[1:2], s[2:3] offset:-128
+; GFX12-NEXT: global_atomic_min_u64 v0, v[1:2], s[2:3] offset:-128 scope:SCOPE_SE
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
deleted file mode 100644
index 0ad16159badf1..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
+++ /dev/null
@@ -1,2759 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX9
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX942
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W32
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX10,GFX10-W64
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX12,GFX12-W32
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX12,GFX12-W64
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -stop-after=si-memory-legalizer < %s -o - | FileCheck %s --check-prefixes=GFX1250
-
-
-define amdgpu_kernel void @wg_fence_acq_rel_single32() #0 {
- ; GFX9-LABEL: name: wg_fence_acq_rel_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acq_rel_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_acq_rel_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_acq_rel_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acq_rel_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_ENDPGM 0
- fence syncscope("workgroup") acq_rel
- ret void
-}
-
-define amdgpu_kernel void @wg_fence_acq_rel_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_acq_rel_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acq_rel_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_fence_acq_rel_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_fence_acq_rel_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_fence_acq_rel_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_fence_acq_rel_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acq_rel_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- fence syncscope("workgroup") acq_rel
- ret void
-}
-
-define amdgpu_kernel void @wg_fence_acq_rel_multi() #2 {
- ; GFX9-LABEL: name: wg_fence_acq_rel_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acq_rel_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_fence_acq_rel_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_fence_acq_rel_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acq_rel_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- fence syncscope("workgroup") acq_rel
- ret void
-}
-
-define amdgpu_kernel void @wg_fence_acquire_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_fence_acquire_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_fence_acquire_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_fence_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_fence_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- fence syncscope("workgroup") acquire
- ret void
-}
-
-define amdgpu_kernel void @wg_fence_release_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_fence_release_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_fence_release_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_fence_release_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_fence_release_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_release_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- fence syncscope("workgroup") release
- ret void
-}
-
-define amdgpu_kernel void @wg_fence_seq_cst_single64() #1 {
- ; GFX9-LABEL: name: wg_fence_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_fence_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_fence_seq_cst_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_fence_seq_cst_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_fence_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_fence_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_fence_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- fence syncscope("workgroup") seq_cst
- ret void
-}
-
-define amdgpu_kernel void @wg_ld_seq_cst_single32(ptr addrspace(1) %p) #0 {
- ; GFX9-LABEL: name: wg_ld_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_seq_cst_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_seq_cst_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_ld_seq_cst_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_ld_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_ld_seq_cst_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_ld_seq_cst_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_ld_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_ld_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_ld_seq_cst_multi(ptr addrspace(1) %p) #2 {
- ; GFX9-LABEL: name: wg_ld_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_seq_cst_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_ld_acquire_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_ld_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_ld_acquire_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_ld_acquire_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_ld_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W32-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_ld_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") acquire (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") acquire, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_ld_monotonic_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_ld_monotonic_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_ld_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_ld_monotonic_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 1, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: wg_ld_monotonic_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 8, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_ld_monotonic_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = GLOBAL_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec :: ("amdgpu-noclobber" load syncscope("workgroup") monotonic (s32) from %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") monotonic, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #0 {
- ; GFX9-LABEL: name: wg_st_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_seq_cst_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_seq_cst_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_seq_cst_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
- store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: wg_st_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_st_seq_cst_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_st_seq_cst_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
- store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
- ; GFX9-LABEL: name: wg_st_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_st_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
- store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: wg_st_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_st_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_st_release_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_st_release_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_st_release_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_st_release_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_st_release_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_ENDPGM 0
- store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") release, align 4
- ret void
-}
-
-define amdgpu_kernel void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) %p) #0 {
- ; GFX9-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
- ret void
-}
-
-define amdgpu_kernel void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) %p) #1 {
- ; GFX9-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
- ret void
-}
-
-define amdgpu_kernel void @wg_rmw_add_seq_cst_multi(ptr addrspace(1) %p) #2 {
- ; GFX9-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W64-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W64-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_add_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: early-clobber renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 7
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr0, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_ADD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr2_sgpr3, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
- ret void
-}
-
-define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_rmw_xchg_acq_rel_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = atomicrmw xchg ptr addrspace(1) %p, i32 %x syncscope("workgroup") acq_rel
- ret void
-}
-
-define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
- ret void
-}
-
-define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p, i32 %cmp, i32 %new) #2 {
- ; GFX9-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") seq_cst seq_cst
- ret void
-}
-
-define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: wg_cmpxchg_acquire_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acquire acquire
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) %p) #0 {
- ; GFX9-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_single32
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) %p) #1 {
- ; GFX9-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_ld_seq_cst_multi(ptr addrspace(3) %p) #2 {
- ; GFX9-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX10-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX12-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: dead renamable $vgpr0 = DS_READ_B32_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load syncscope("workgroup") seq_cst (s32) from %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_st_release_single64(ptr addrspace(3) %p, i32 %x) #1 {
- ; GFX9-LABEL: name: lds_wg_st_release_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_st_release_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_wg_st_release_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_wg_st_release_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_st_release_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_st_release_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_st_release_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 3)
- ; GFX1250-NEXT: S_ENDPGM 0
- store atomic i32 %x, ptr addrspace(3) %p syncscope("workgroup") release, align 4
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) %p) #1 {
- ; GFX9-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX9-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX9-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX9-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX9-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.1 (%ir-block.11):
- ; GFX9-NEXT: successors: %bb.2(0x80000000)
- ; GFX9-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX9-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: bb.2 (%ir-block.16):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX942-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX942-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX942-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX942-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.1 (%ir-block.11):
- ; GFX942-NEXT: successors: %bb.2(0x80000000)
- ; GFX942-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX942-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.2 (%ir-block.16):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX10-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W32-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
- ; GFX10-W32-NEXT: $sgpr1 = S_AND_SAVEEXEC_B32 killed $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX10-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX10-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX10-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX10-W64-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX10-W64-NEXT: $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX10-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX10-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX10-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX10-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX12-W32-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W32-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W32-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.1 (%ir-block.7):
- ; GFX12-W32-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W32-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX12-W32-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX12-W32-NEXT: $vgpr1, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: bb.2 (%ir-block.11):
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: $sgpr2_sgpr3 = S_MOV_B64 $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX12-W64-NEXT: renamable $vgpr0 = V_MBCNT_HI_U32_B32_e64 $sgpr1, killed $vgpr0, implicit $exec
- ; GFX12-W64-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX12-W64-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.1 (%ir-block.11):
- ; GFX12-W64-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-W64-NEXT: liveins: $sgpr0_sgpr1:0x000000000000000F, $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_BCNT1_I32_B64 killed renamable $sgpr0_sgpr1, implicit-def dead $scc
- ; GFX12-W64-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: bb.2 (%ir-block.16):
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_rmw_add_acq_rel_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: $sgpr0 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: $sgpr1 = S_MOV_B32 $exec_lo
- ; GFX1250-NEXT: renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 $sgpr0, 0, implicit $exec
- ; GFX1250-NEXT: V_CMPX_EQ_U32_nosdst_e32 0, killed $vgpr0, implicit-def $exec, implicit $exec
- ; GFX1250-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.1 (%ir-block.7):
- ; GFX1250-NEXT: successors: %bb.2(0x80000000)
- ; GFX1250-NEXT: liveins: $sgpr0, $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s32) from %ir.p.kernarg.offset, addrspace 4)
- ; GFX1250-NEXT: renamable $sgpr0 = S_BCNT1_I32_B32 killed renamable $sgpr0, implicit-def dead $scc
- ; GFX1250-NEXT: renamable $sgpr0 = S_MUL_I32 killed renamable $sgpr0, 3
- ; GFX1250-NEXT: $vgpr1, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_ADD_U32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: bb.2 (%ir-block.11):
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = atomicrmw add ptr addrspace(3) %p, i32 3 syncscope("workgroup") acq_rel
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-W32-NEXT: S_WAITCNT_lds_direct
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_cmpxchg_acq_rel_monotonic_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = cmpxchg ptr addrspace(3) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_ld_unordered_single64(ptr addrspace(3) %p) #1 {
- ; GFX9-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX12: bb.0 (%ir-block.0):
- ; GFX12-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_ld_unordered_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(3) %p syncscope("workgroup") unordered, align 4
- ret void
-}
-
-define amdgpu_kernel void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) %p, i32 %cmp, i32 %new) #1 {
- ; GFX9-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: DS_CMPST_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: lds_wg_cmpxchg_monotonic_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 killed $sgpr0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
- ; GFX1250-NEXT: DS_CMPSTORE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec :: (load store syncscope("workgroup") monotonic acquire (s32) on %ir.2, addrspace 3)
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %r = cmpxchg ptr addrspace(3) %p, i32 %cmp, i32 %new syncscope("workgroup") monotonic acquire
- ret void
-}
-
-define amdgpu_kernel void @flat_wg_ld_acquire_single64(ptr addrspace(0) %p) #1 {
- ; GFX9-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX9-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W32-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX10-W32: bb.0 (%ir-block.0):
- ; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX10-W32-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 1, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
- ; GFX10-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX10-W64-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX10-W64: bb.0 (%ir-block.0):
- ; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX10-W64-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX10-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W32-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 8, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W64-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: flat_wg_ld_acquire_single64
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
- ; GFX1250-NEXT: dead renamable $vgpr0 = FLAT_LOAD_DWORD_SADDR killed renamable $sgpr0_sgpr1, killed renamable $vgpr0, 0, 0, implicit $exec, implicit $flat_scr :: (load syncscope("workgroup") acquire (s32) from %ir.p.load)
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_ENDPGM 0
- %v = load atomic i32, ptr addrspace(0) %p syncscope("workgroup") acquire, align 4
- ret void
-}
-
-define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x) #2 {
- ; GFX9-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX9: bb.0 (%ir-block.0):
- ; GFX9-NEXT: liveins: $sgpr4_sgpr5
- ; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX9-NEXT: S_ENDPGM 0
- ;
- ; GFX942-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX942: bb.0 (%ir-block.0):
- ; GFX942-NEXT: liveins: $sgpr4_sgpr5
- ; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
- ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
- ; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 1, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX942-NEXT: S_ENDPGM 0
- ;
- ; GFX10-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX10: bb.0 (%ir-block.0):
- ; GFX10-NEXT: liveins: $sgpr4_sgpr5
- ; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
- ; GFX10-NEXT: S_WAITCNT_lds_direct
- ; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX10-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W32-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX12-W32: bb.0 (%ir-block.0):
- ; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W32-NEXT: {{ $}}
- ; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX12-W32-NEXT: S_ENDPGM 0
- ;
- ; GFX12-W64-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX12-W64: bb.0 (%ir-block.0):
- ; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
- ; GFX12-W64-NEXT: {{ $}}
- ; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit $sgpr0_sgpr1
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- ; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX12-W64-NEXT: S_ENDPGM 0
- ;
- ; GFX1250-LABEL: name: flat_wg_st_seq_cst_multi
- ; GFX1250: bb.0 (%ir-block.0):
- ; GFX1250-NEXT: liveins: $sgpr4_sgpr5
- ; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: S_SETREG_IMM32_B32 1, 1601, implicit-def $mode, implicit $mode
- ; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX1250-NEXT: S_WAIT_LOADCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
- ; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: FLAT_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
- ; GFX1250-NEXT: S_ENDPGM 0
- store atomic i32 %x, ptr addrspace(0) %p syncscope("workgroup") seq_cst, align 4
- ret void
-}
-
-attributes #0 = { "amdgpu-flat-work-group-size"="32,32" nounwind }
-attributes #1 = { "amdgpu-flat-work-group-size"="64,64" nounwind }
-attributes #2 = { "amdgpu-flat-work-group-size"="64,256" nounwind }
More information about the llvm-commits
mailing list