[llvm-branch-commits] [llvm] [AMDGPU][SIMemoryLegalizer] Lower single-wave workgroup scope to wavefront in absence of LDSDMA (PR #224065)
Zach Goldthorpe via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 7 10:51:12 PDT 2026
https://github.com/zGoldthorpe updated https://github.com/llvm/llvm-project/pull/224065
>From c8d992ac8d44cccb623ef1afeef05b1a8be5121d Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Mon, 5 Oct 2026 10:58:51 -0500
Subject: [PATCH 1/2] Re-apply demotion logic
This reverts commit 992e6130ed441b82490f9a6cf01f5bba38293e4e.
---
llvm/docs/AMDGPUUsage.rst | 10 +
.../Target/AMDGPU/AMDGPULowerIntrinsics.cpp | 6 +-
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 4 +
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h | 4 +
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 27 +-
...zer-single-wave-workgroup-lds-dma-async.ll | 7 +-
...legalizer-single-wave-workgroup-lds-dma.ll | 60 +-
...-legalizer-single-wave-workgroup-memops.ll | 2226 ++++++++++-------
8 files changed, 1439 insertions(+), 905 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index bba48a4d56457..989de7facbd58 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -7553,6 +7553,16 @@ treated as non-atomic.
A memory synchronization scope wider than work-group is not meaningful for the
group (LDS) address space and is treated as work-group.
+When a work-group's maximum flat work-group size does not exceed the wavefront
+size, the work-group fits within a single wavefront. So long as no asynchronous
+operations occur, the LLVM ``workgroup`` synchronization scope is equivalent to
+its ``wavefront`` scope.
+
+If the compiler can determine these conditions (e.g., through the function attributes
+``amdgpu-flat-work-group-size`` and ``amdgpu-no-async``), the AMDGPU backend
+optimizes ``workgroup`` scope operations by lowering them to
+``wavefront``-scoped machine instructions.
+
The memory model does not support the region address space which is treated as
non-atomic.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index 6150e14f38c7d..83ea136ab9fef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -105,10 +105,8 @@ bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(*I.getFunction());
bool IsSingleWaveWG = false;
- if (TM.getOptLevel() > CodeGenOptLevel::None) {
- unsigned WGMaxSize = ST.getFlatWorkGroupSizes(*I.getFunction()).second;
- IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
- }
+ if (TM.getOptLevel() > CodeGenOptLevel::None)
+ IsSingleWaveWG = ST.isSingleWavefrontWorkgroup(*I.getFunction());
IRBuilder<> B(&I);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index b9ba99cf8db19..ca78fa5911084 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -177,6 +177,10 @@ std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
return Requested;
}
+bool AMDGPUSubtarget::isSingleWavefrontWorkgroup(const Function &F) const {
+ return getFlatWorkGroupSizes(F).second <= getWavefrontSize();
+}
+
std::pair<unsigned, unsigned> AMDGPUSubtarget::getEffectiveWavesPerEU(
std::pair<unsigned, unsigned> RequestedWavesPerEU,
std::pair<unsigned, unsigned> FlatWorkGroupSizes, unsigned LDSBytes) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index e1f331229c463..4fa6020e9198c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -82,6 +82,10 @@ class AMDGPUSubtarget {
/// be converted to integer, or violate subtarget's specifications.
std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
+ /// \returns true if the maximum flat work-group size for \p F is at most the
+ /// wavefront size, so a work-group may fit in a single wavefront.
+ bool isSingleWavefrontWorkgroup(const Function &F) const;
+
/// \returns The required size of workgroups that will be used to execute \p F
/// in the \p Dim dimension, if it is known (from `!reqd_work_group_size`
/// metadata. Otherwise, returns std::nullopt.
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index e818e06e3bb04..7f182520c08e7 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -159,7 +159,8 @@ class SIMemOpInfo final {
bool IsCrossAddressSpaceOrdering = true,
AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent,
bool IsVolatile = false, bool IsNonTemporal = false,
- bool IsLastUse = false, bool IsCooperative = false, bool IsAVNone = false)
+ bool IsLastUse = false, bool IsCooperative = false, bool IsAVNone = false,
+ bool CanDemoteWorkgroupToWavefront = false)
: Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope),
OrderingAddrSpace(OrderingAddrSpace), InstrAddrSpace(InstrAddrSpace),
IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
@@ -207,6 +208,10 @@ class SIMemOpInfo final {
// AGENT scope as a conservatively correct alternative.
if (this->Scope == SIAtomicScope::CLUSTER && !ST.hasClusters())
this->Scope = SIAtomicScope::AGENT;
+
+ if (CanDemoteWorkgroupToWavefront &&
+ this->Scope == SIAtomicScope::WORKGROUP)
+ this->Scope = SIAtomicScope::WAVEFRONT;
}
public:
@@ -280,6 +285,7 @@ class SIMemOpAccess final {
private:
const AMDGPUMachineModuleInfo *MMI = nullptr;
const GCNSubtarget &ST;
+ const bool CanDemoteWorkgroupToWavefront;
/// Reports unsupported message \p Msg for \p MI to LLVM context.
void reportUnsupported(const MachineBasicBlock::iterator &MI,
@@ -303,7 +309,8 @@ class SIMemOpAccess final {
public:
/// Construct class to support accessing the machine memory operands
/// of instructions.
- SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST);
+ SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST,
+ const MachineFunction &MF);
/// \returns Load info if \p MI is a load operation, "std::nullopt" otherwise.
std::optional<SIMemOpInfo>
@@ -824,9 +831,15 @@ SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const {
return SIAtomicAddrSpace::OTHER;
}
+// TODO: Consider moving single-wave workgroup->wavefront scope relaxation to an
+// IR pass (and extending it to other scoped operations), so middle-end
+// optimizations see wavefront scope earlier.
SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_,
- const GCNSubtarget &ST)
- : MMI(&MMI_), ST(ST) {}
+ const GCNSubtarget &ST, const MachineFunction &MF)
+ : MMI(&MMI_), ST(ST),
+ CanDemoteWorkgroupToWavefront(
+ ST.isSingleWavefrontWorkgroup(MF.getFunction()) &&
+ MF.getFunction().hasFnAttribute("amdgpu-no-async")) {}
std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
const MachineBasicBlock::iterator &MI) const {
@@ -898,7 +911,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
IsNonTemporal, IsLastUse, IsCooperative,
- hasAVNoneMMRA(*MI));
+ hasAVNoneMMRA(*MI), CanDemoteWorkgroupToWavefront);
}
std::optional<SIMemOpInfo>
@@ -968,7 +981,7 @@ SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const {
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace,
SIAtomicAddrSpace::ATOMIC, IsCrossAddressSpaceOrdering,
AtomicOrdering::NotAtomic, false, false, false, false,
- hasAVNoneMMRA(*MI));
+ hasAVNoneMMRA(*MI), CanDemoteWorkgroupToWavefront);
}
std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
@@ -2586,7 +2599,7 @@ bool SIMemoryLegalizer::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const Function &F = MF.getFunction();
- SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>(), ST);
+ SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>(), ST, MF);
bool TgSplit = ST.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F);
CC = SICacheControl::create(ST, TgSplit);
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
index 92ee71e12937c..e8657b8a589fd 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma-async.ll
@@ -11,8 +11,6 @@ define void @lds_wg_fence_release_single32(ptr addrspace(3) inreg %lds) #0 {
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ds_store_b32 v0, v1
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ds_load_b32 v1, v0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
@@ -27,7 +25,7 @@ define void @lds_wg_fence_release_single32(ptr addrspace(3) inreg %lds) #0 {
ret void
}
-define void @lds_async_dma_wg_fence_release_single32(ptr addrspace(1) inreg %g, ptr addrspace(3) inreg %lds) #0 {
+define void @lds_async_dma_wg_fence_release_single32(ptr addrspace(1) inreg %g, ptr addrspace(3) inreg %lds) #1 {
; GFX1250-LABEL: lds_async_dma_wg_fence_release_single32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -54,4 +52,5 @@ define void @lds_async_dma_wg_fence_release_single32(ptr addrspace(1) inreg %g,
ret void
}
-attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" "amdgpu-no-async" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
index 06e0af551d1aa..250a28a05cfd4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-lds-dma.ll
@@ -13,7 +13,6 @@ define void @lds_wg_fence_release_single32(ptr addrspace(3) inreg %lds) #0 {
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_read_b32 v1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -28,7 +27,6 @@ define void @lds_wg_fence_release_single32(ptr addrspace(3) inreg %lds) #0 {
; GFX942-NEXT: v_mov_b32_e32 v1, 1
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_read_b32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
@@ -43,8 +41,6 @@ define void @lds_wg_fence_release_single32(ptr addrspace(3) inreg %lds) #0 {
; GFX10-NEXT: v_mov_b32_e32 v1, 1
; GFX10-NEXT: v_mov_b32_e32 v0, s16
; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, s16
; GFX10-NEXT: ds_read_b32 v1, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s16
@@ -66,7 +62,6 @@ define void @lds_wg_fence_release_single64(ptr addrspace(3) inreg %lds) #1 {
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_read_b32 v1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -81,7 +76,6 @@ define void @lds_wg_fence_release_single64(ptr addrspace(3) inreg %lds) #1 {
; GFX942-NEXT: v_mov_b32_e32 v1, 1
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_read_b32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
@@ -90,21 +84,35 @@ define void @lds_wg_fence_release_single64(ptr addrspace(3) inreg %lds) #1 {
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_fence_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, 1
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_read_b32 v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_fence_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 1
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_read_b32 v1, v0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_fence_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 1
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_read_b32 v1, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
store i32 1, ptr addrspace(3) %lds, align 4
fence syncscope("workgroup") release
%v = load i32, ptr addrspace(3) %lds, align 4
@@ -112,7 +120,7 @@ define void @lds_wg_fence_release_single64(ptr addrspace(3) inreg %lds) #1 {
ret void
}
-define void @lds_dma_wg_fence_release_single32(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) inreg %lds) #0 {
+define void @lds_dma_wg_fence_release_single32(ptr addrspace(8) inreg %rsrc, ptr addrspace(3) inreg %lds) #2 {
; GFX9-LABEL: lds_dma_wg_fence_release_single32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -209,8 +217,6 @@ define void @lds_dma_wg_fence_release_single32(ptr addrspace(8) inreg %rsrc, ptr
ret void
}
-attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
-attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-W32: {{.*}}
-; GFX10-W64: {{.*}}
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" "amdgpu-no-async" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-async" }
+attributes #2 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
index 30c6443d8c474..e3227af40369f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
@@ -17,7 +17,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -33,7 +32,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
@@ -49,9 +47,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: v_mov_b32_e32 v1, s18
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
@@ -71,11 +66,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: v_mov_b32_e32 v1, s2
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -92,8 +82,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, s2
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -115,7 +103,6 @@ define void @wg_fence_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -131,54 +118,87 @@ define void @wg_fence_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_acq_rel_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_acq_rel_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_acq_rel_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_acq_rel_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_acq_rel_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_acq_rel_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_acq_rel_single64:
; GFX1250: ; %bb.0:
@@ -311,7 +331,6 @@ define void @wg_fence_acquire_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -327,52 +346,85 @@ define void @wg_fence_acquire_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_acquire_single64:
; GFX1250: ; %bb.0:
@@ -407,7 +459,6 @@ define void @wg_fence_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -423,52 +474,85 @@ define void @wg_fence_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_release_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_release_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_release_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_release_single64:
; GFX1250: ; %bb.0:
@@ -503,7 +587,6 @@ define void @wg_fence_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -519,54 +602,87 @@ define void @wg_fence_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -599,7 +715,6 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX9-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
@@ -613,8 +728,7 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -627,12 +741,9 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: wg_ld_seq_cst_single32:
@@ -647,14 +758,9 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -666,11 +772,9 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX1250-NEXT: s_mov_b32 s1, s2
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
store i32 0, ptr addrspace(1) %p, align 4
@@ -685,7 +789,6 @@ define void @wg_ld_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX9-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
@@ -699,50 +802,79 @@ define void @wg_ld_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_ld_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_ld_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_ld_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_ld_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_ld_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_ld_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_ld_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -870,43 +1002,73 @@ define void @wg_ld_acquire_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_ld_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_ld_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_ld_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_ld_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_ld_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_ld_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_ld_acquire_single64:
; GFX1250: ; %bb.0:
@@ -946,7 +1108,7 @@ define void @wg_ld_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -959,7 +1121,7 @@ define void @wg_ld_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -976,7 +1138,7 @@ define void @wg_ld_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1020,41 +1182,71 @@ define void @wg_ld_monotonic_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_ld_monotonic_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_ld_monotonic_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_ld_monotonic_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_ld_monotonic_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_ld_monotonic_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_ld_monotonic_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_ld_monotonic_single64:
; GFX1250: ; %bb.0:
@@ -1156,7 +1348,6 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1170,8 +1361,7 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1184,8 +1374,6 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1203,11 +1391,7 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX12-NEXT: s_mov_b32 s1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1221,9 +1405,7 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX1250-NEXT: s_mov_b32 s1, s3
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, s2
-; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1250-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -1241,7 +1423,6 @@ define void @wg_st_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1255,47 +1436,76 @@ define void @wg_st_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: wg_st_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: wg_st_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_st_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_st_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_st_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_st_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_st_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -1411,7 +1621,6 @@ define void @wg_st_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1425,47 +1634,76 @@ define void @wg_st_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_st_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_st_release_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_st_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_st_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_st_release_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_st_release_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_st_release_single64:
; GFX1250: ; %bb.0:
@@ -1509,7 +1747,7 @@ define void @wg_st_monotonic_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1539,7 +1777,7 @@ define void @wg_st_monotonic_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #
; GFX12-NEXT: s_mov_b32 s1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1646,7 +1884,6 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 7
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_add v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1660,7 +1897,6 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v1, 7
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1674,11 +1910,7 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v1, 7
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_add v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -1695,13 +1927,7 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v1, 7
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1715,11 +1941,8 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX1250-NEXT: s_mov_b32 s1, s2
; GFX1250-NEXT: v_mov_b32_e32 v1, 7
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
-; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
@@ -1736,7 +1959,6 @@ define void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 7
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_add v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1750,51 +1972,80 @@ define void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v1, 7
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_rmw_add_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v1, 7
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_add v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_rmw_add_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v1, 7
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_atomic_add v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_atomic_add v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_rmw_add_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -1916,7 +2167,6 @@ define void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_swap v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1930,51 +2180,80 @@ define void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_swap v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_rmw_xchg_acq_rel_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_swap v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_rmw_xchg_acq_rel_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_atomic_swap v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_atomic_swap v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_rmw_xchg_acq_rel_single64:
; GFX1250: ; %bb.0:
@@ -2049,7 +2328,7 @@ define void @wg_rmw_add_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v1, 7
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -2159,7 +2438,6 @@ define void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) inreg %p, i3
; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2176,57 +2454,92 @@ define void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) inreg %p, i3
; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v1, s19
-; GFX10-NEXT: v_mov_b32_e32 v0, s18
-; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s4
-; GFX12-NEXT: v_mov_b32_e32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s4, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s4
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s4, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s4
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
; GFX1250: ; %bb.0:
@@ -2372,63 +2685,99 @@ define void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1) inreg %p, i32
; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: wg_cmpxchg_acquire_acquire_single64:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s4, s1
-; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX942-NEXT: s_mov_b32 s1, s4
-; GFX942-NEXT: v_mov_b32_e32 v2, s3
-; GFX942-NEXT: v_mov_b32_e32 v0, s2
-; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
-; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: wg_cmpxchg_acquire_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v1, s19
-; GFX10-NEXT: v_mov_b32_e32 v0, s18
-; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_cmpxchg_acquire_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s4
-; GFX12-NEXT: v_mov_b32_e32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, s1
+; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX942-NEXT: s_mov_b32 s1, s4
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v0, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W32-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s4, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s4
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s4, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s4
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_cmpxchg_acquire_acquire_single64:
; GFX1250: ; %bb.0:
@@ -2516,7 +2865,7 @@ define void @wg_cmpxchg_monotonic_monotonic_single32(ptr addrspace(1) inreg %p,
; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-NEXT: v_mov_b32_e32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -2637,10 +2986,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_read_b32 v0, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2650,10 +2998,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v0, v0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
@@ -2663,12 +3010,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_read_b32 v0, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s16
; GFX10-NEXT: ds_write_b32 v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -2682,14 +3026,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ds_store_b32 v0, v1
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -2700,11 +3039,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ds_load_b32 v0, v0
-; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ds_store_b32 v0, v1
; GFX1250-NEXT: s_wait_dscnt 0x0
@@ -2719,10 +3056,9 @@ define void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) inreg %p) #1 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_read_b32 v0, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2732,50 +3068,77 @@ define void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) inreg %p) #1 {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v0, v0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_ld_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_read_b32 v0, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_ld_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_read_b32 v0, v0
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_read_b32 v0, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_load_b32 v0, v0
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_load_b32 v0, v0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_ld_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -2884,7 +3247,6 @@ define void @lds_wg_st_release_single64(ptr addrspace(3) inreg %p, i32 inreg %x)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_write_b32 v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -2897,7 +3259,6 @@ define void @lds_wg_st_release_single64(ptr addrspace(3) inreg %p, i32 inreg %x)
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: v_mov_b32_e32 v1, s1
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
@@ -2905,39 +3266,67 @@ define void @lds_wg_st_release_single64(ptr addrspace(3) inreg %p, i32 inreg %x)
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_st_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: v_mov_b32_e32 v1, s17
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_st_release_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s1
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_st_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_st_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_st_release_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_st_release_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_st_release_single64:
; GFX1250: ; %bb.0:
@@ -2964,9 +3353,7 @@ define void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) inreg %p) #1 {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s16
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_add_u32 v0, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
@@ -2978,52 +3365,78 @@ define void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) inreg %p) #1 {
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 3
; GFX942-NEXT: v_mov_b32_e32 v0, s0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_add_u32 v0, v1
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_rmw_add_acq_rel_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, 3
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_add_u32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_rmw_add_acq_rel_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, 3
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_add_u32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 3
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_add_u32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 3
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_add_u32 v0, v1
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_add_u32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_add_u32 v0, v1
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_rmw_add_acq_rel_single64:
; GFX1250: ; %bb.0:
@@ -3052,9 +3465,7 @@ define void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) inreg %p
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
@@ -3067,54 +3478,82 @@ define void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) inreg %p
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: v_mov_b32_e32 v1, s1
; GFX942-NEXT: v_mov_b32_e32 v2, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: v_mov_b32_e32 v1, s17
-; GFX10-NEXT: v_mov_b32_e32 v2, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v2, s1
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_cmpstore_b32 v0, v1, v2
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W64-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W64-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
; GFX1250: ; %bb.0:
@@ -3215,7 +3654,6 @@ define void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) inreg %p
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
@@ -3229,46 +3667,75 @@ define void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) inreg %p
; GFX942-NEXT: v_mov_b32_e32 v1, s1
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: v_mov_b32_e32 v1, s17
-; GFX10-NEXT: v_mov_b32_e32 v2, s18
-; GFX10-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v2, s1
-; GFX12-NEXT: ds_cmpstore_b32 v0, v1, v2
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W32-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W64-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W32-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W64-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
; GFX1250: ; %bb.0:
@@ -3299,9 +3766,8 @@ define void @flat_wg_ld_acquire_single64(ptr addrspace(0) inreg %p) #1 {
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dword v0, v[0:1]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_store_dword v[0:1], v2
@@ -3315,56 +3781,95 @@ define void @flat_wg_ld_acquire_single64(ptr addrspace(0) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-NEXT: flat_load_dword v0, v[0:1] sc0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: flat_load_dword v0, v[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: flat_store_dword v[0:1], v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_wg_ld_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: flat_load_dword v0, v[0:1] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: flat_store_dword v[0:1], v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: flat_wg_ld_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s1
-; GFX12-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s1
-; GFX12-NEXT: flat_store_b32 v[0:1], v2
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: flat_wg_ld_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W32-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W32-NEXT: flat_store_dword v[0:1], v2
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: flat_wg_ld_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W64-NEXT: flat_load_dword v0, v[0:1]
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W64-NEXT: flat_store_dword v[0:1], v2
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: flat_wg_ld_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W32-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W32-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: flat_wg_ld_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W64-NEXT: flat_load_b32 v0, v[0:1]
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W64-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: flat_wg_ld_acquire_single64:
; GFX1250: ; %bb.0:
@@ -3486,11 +3991,6 @@ define void @flat_wg_st_seq_cst_multi(ptr addrspace(0) inreg %p, i32 inreg %x) #
ret void
}
-attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" }
-attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" }
-attributes #2 = { nounwind "amdgpu-flat-work-group-size"="64,256" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-W32: {{.*}}
-; GFX10-W64: {{.*}}
-; GFX12-W32: {{.*}}
-; GFX12-W64: {{.*}}
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" "amdgpu-no-async" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-async" }
+attributes #2 = { nounwind "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-async" }
>From 0c3e6b698156fa2d0aa44ae0904113b5480ce0a6 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Wed, 7 Oct 2026 10:56:09 -0500
Subject: [PATCH 2/2] Update attributor name
---
llvm/docs/AMDGPUUsage.rst | 8 +-
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 2 +-
...-legalizer-single-wave-workgroup-memops.ll | 2220 ++++++++++-------
3 files changed, 1365 insertions(+), 865 deletions(-)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index cc23f924ce799..c794cb59e3815 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -7554,13 +7554,13 @@ A memory synchronization scope wider than work-group is not meaningful for the
group (LDS) address space and is treated as work-group.
When a work-group's maximum flat work-group size does not exceed the wavefront
-size, the work-group fits within a single wavefront. So long as no asynchronous
+size, the work-group fits within a single wavefront. So long as no LDSDMA
operations occur, the LLVM ``workgroup`` synchronization scope is equivalent to
its ``wavefront`` scope.
-If the compiler can determine these conditions (e.g., through the function attributes
-``amdgpu-flat-work-group-size`` and ``amdgpu-no-async``), the AMDGPU backend
-optimizes ``workgroup`` scope operations by lowering them to
+If the compiler can determine these conditions (e.g., through the function
+attributes ``amdgpu-flat-work-group-size`` and ``amdgpu-no-lds-dma``), the
+AMDGPU backend optimizes ``workgroup`` scope operations by lowering them to
``wavefront``-scoped machine instructions.
The memory model does not support the region address space which is treated as
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 7f182520c08e7..3f5f11d155fd4 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -839,7 +839,7 @@ SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_,
: MMI(&MMI_), ST(ST),
CanDemoteWorkgroupToWavefront(
ST.isSingleWavefrontWorkgroup(MF.getFunction()) &&
- MF.getFunction().hasFnAttribute("amdgpu-no-async")) {}
+ MF.getFunction().hasFnAttribute("amdgpu-no-lds-dma")) {}
std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
const MachineBasicBlock::iterator &MI) const {
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
index 894a7b860957b..b3c6bfa3e890e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
@@ -17,7 +17,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -33,7 +32,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
@@ -49,9 +47,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: v_mov_b32_e32 v1, s18
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
@@ -71,11 +66,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: v_mov_b32_e32 v1, s2
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -92,8 +82,6 @@ define void @wg_fence_acq_rel_single32(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, s2
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -115,7 +103,6 @@ define void @wg_fence_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -131,54 +118,87 @@ define void @wg_fence_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_acq_rel_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_acq_rel_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_acq_rel_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_acq_rel_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_acq_rel_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_acq_rel_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_acq_rel_single64:
; GFX1250: ; %bb.0:
@@ -311,7 +331,6 @@ define void @wg_fence_acquire_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -327,52 +346,85 @@ define void @wg_fence_acquire_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_acquire_single64:
; GFX1250: ; %bb.0:
@@ -407,7 +459,6 @@ define void @wg_fence_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -423,52 +474,85 @@ define void @wg_fence_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_release_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_release_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_release_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_release_single64:
; GFX1250: ; %bb.0:
@@ -503,7 +587,6 @@ define void @wg_fence_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
@@ -519,54 +602,87 @@ define void @wg_fence_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x)
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_fence_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_fence_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_fence_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_fence_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_fence_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_fence_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_fence_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -599,7 +715,6 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX9-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
@@ -613,8 +728,7 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -627,12 +741,9 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: wg_ld_seq_cst_single32:
@@ -647,14 +758,9 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
@@ -666,11 +772,9 @@ define void @wg_ld_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX1250-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX1250-NEXT: s_mov_b32 s1, s2
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%v = load atomic i32, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
store i32 0, ptr addrspace(1) %p, align 4
@@ -685,7 +789,6 @@ define void @wg_ld_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX9-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[4:5]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
@@ -699,50 +802,79 @@ define void @wg_ld_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_ld_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_ld_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_ld_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_ld_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_ld_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_ld_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_ld_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -870,43 +1002,73 @@ define void @wg_ld_acquire_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_ld_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_ld_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_ld_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_ld_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_ld_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_ld_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_ld_acquire_single64:
; GFX1250: ; %bb.0:
@@ -946,7 +1108,7 @@ define void @wg_ld_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -959,7 +1121,7 @@ define void @wg_ld_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-NEXT: global_load_dword v1, v0, s[4:5]
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -976,7 +1138,7 @@ define void @wg_ld_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1020,41 +1182,71 @@ define void @wg_ld_monotonic_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_load_dword v1, v0, s[0:1] sc0
+; GFX942-NEXT: global_load_dword v1, v0, s[0:1]
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_ld_monotonic_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_load_dword v1, v0, s[4:5] glc
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_ld_monotonic_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_ld_monotonic_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: global_load_dword v1, v0, s[4:5] glc
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_ld_monotonic_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_load_dword v1, v0, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_ld_monotonic_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: global_load_b32 v1, v0, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_loadcnt 0x0
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_ld_monotonic_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_load_b32 v1, v0, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_loadcnt 0x0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_ld_monotonic_single64:
; GFX1250: ; %bb.0:
@@ -1156,7 +1348,6 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1170,8 +1361,7 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1184,8 +1374,6 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -1203,11 +1391,7 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX12-NEXT: s_mov_b32 s1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1221,9 +1405,7 @@ define void @wg_st_seq_cst_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #0
; GFX1250-NEXT: s_mov_b32 s1, s3
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, s2
-; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1250-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -1241,7 +1423,6 @@ define void @wg_st_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1255,47 +1436,76 @@ define void @wg_st_seq_cst_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: wg_st_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: wg_st_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_st_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_st_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_st_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_st_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_st_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -1411,7 +1621,6 @@ define void @wg_st_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1425,47 +1634,76 @@ define void @wg_st_release_single64(ptr addrspace(1) inreg %p, i32 inreg %x) #1
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_st_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_st_release_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_st_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_st_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_st_release_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_st_release_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_st_release_single64:
; GFX1250: ; %bb.0:
@@ -1509,7 +1747,7 @@ define void @wg_st_monotonic_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: global_store_dword v0, v1, s[0:1] sc0
+; GFX942-NEXT: global_store_dword v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1539,7 +1777,7 @@ define void @wg_st_monotonic_single32(ptr addrspace(1) inreg %p, i32 inreg %x) #
; GFX12-NEXT: s_mov_b32 s1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, 0
; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1646,7 +1884,6 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 7
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_add v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1660,7 +1897,6 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v1, 7
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1674,11 +1910,7 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX10-NEXT: s_mov_b32 s5, s17
; GFX10-NEXT: v_mov_b32_e32 v1, 7
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_add v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -1695,13 +1927,7 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v1, 7
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -1715,11 +1941,8 @@ define void @wg_rmw_add_seq_cst_single32(ptr addrspace(1) inreg %p) #0 {
; GFX1250-NEXT: s_mov_b32 s1, s2
; GFX1250-NEXT: v_mov_b32_e32 v1, 7
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
-; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%r = atomicrmw add ptr addrspace(1) %p, i32 7 syncscope("workgroup") seq_cst
@@ -1736,7 +1959,6 @@ define void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v1, 7
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_add v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1750,51 +1972,80 @@ define void @wg_rmw_add_seq_cst_single64(ptr addrspace(1) inreg %p) #1 {
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b32_e32 v1, 7
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_add v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_rmw_add_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v1, 7
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_add v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_rmw_add_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v1, 7
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_atomic_add v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 7
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_atomic_add v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_rmw_add_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 7
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_rmw_add_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -1916,7 +2167,6 @@ define void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %
; GFX9-NEXT: s_mov_b32 s5, s17
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_swap v0, v1, s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -1930,51 +2180,80 @@ define void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) inreg %p, i32 inreg %
; GFX942-NEXT: s_mov_b32 s1, s3
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: v_mov_b32_e32 v1, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_swap v0, v1, s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_rmw_xchg_acq_rel_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_swap v0, v1, s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_rmw_xchg_acq_rel_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s3, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_atomic_swap v0, v1, s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s18
+; GFX10-W64-NEXT: global_atomic_swap v0, v1, s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s3, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_rmw_xchg_acq_rel_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s3, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_rmw_xchg_acq_rel_single64:
; GFX1250: ; %bb.0:
@@ -2049,7 +2328,7 @@ define void @wg_rmw_add_monotonic_single32(ptr addrspace(1) inreg %p) #0 {
; GFX12-NEXT: s_mov_b32 s1, s2
; GFX12-NEXT: v_mov_b32_e32 v1, 7
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -2159,7 +2438,6 @@ define void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) inreg %p, i3
; GFX9-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2176,57 +2454,92 @@ define void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1) inreg %p, i3
; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v1, s19
-; GFX10-NEXT: v_mov_b32_e32 v0, s18
-; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s4
-; GFX12-NEXT: v_mov_b32_e32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s4, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s4
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s4, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s4
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_cmpxchg_acq_rel_monotonic_single64:
; GFX1250: ; %bb.0:
@@ -2372,63 +2685,99 @@ define void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1) inreg %p, i32
; GFX9-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: wg_cmpxchg_acquire_acquire_single64:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s4, s1
-; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX942-NEXT: s_mov_b32 s1, s4
-; GFX942-NEXT: v_mov_b32_e32 v2, s3
-; GFX942-NEXT: v_mov_b32_e32 v0, s2
-; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
-; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: wg_cmpxchg_acquire_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v1, s19
-; GFX10-NEXT: v_mov_b32_e32 v0, s18
-; GFX10-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: global_store_dword v0, v0, s[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: wg_cmpxchg_acquire_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s4, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s4
-; GFX12-NEXT: v_mov_b32_e32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX12-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, s1
+; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX942-NEXT: s_mov_b32 s1, s4
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v0, s2
+; GFX942-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1]
+; GFX942-NEXT: global_store_dword v0, v0, s[0:1]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W32-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W32-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s19
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s18
+; GFX10-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-W64-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5]
+; GFX10-W64-NEXT: global_store_dword v0, v0, s[4:5]
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s4, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s4
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W32-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: wg_cmpxchg_acquire_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s4, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s4
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, 0
+; GFX12-W64-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1]
+; GFX12-W64-NEXT: global_store_b32 v0, v0, s[0:1]
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: wg_cmpxchg_acquire_acquire_single64:
; GFX1250: ; %bb.0:
@@ -2516,7 +2865,7 @@ define void @wg_cmpxchg_monotonic_monotonic_single32(ptr addrspace(1) inreg %p,
; GFX12-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX12-NEXT: v_mov_b32_e32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] scope:SCOPE_SE
+; GFX12-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1]
; GFX12-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -2637,10 +2986,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_read_b32 v0, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2650,10 +2998,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v0, v0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
@@ -2663,12 +3010,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_read_b32 v0, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v0, s16
; GFX10-NEXT: ds_write_b32 v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -2682,14 +3026,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ds_store_b32 v0, v1
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -2700,11 +3039,9 @@ define void @lds_wg_ld_seq_cst_single32(ptr addrspace(3) inreg %p) #0 {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ds_load_b32 v0, v0
-; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-NEXT: ds_store_b32 v0, v1
; GFX1250-NEXT: s_wait_dscnt 0x0
@@ -2719,10 +3056,9 @@ define void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) inreg %p) #1 {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_read_b32 v0, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2732,50 +3068,77 @@ define void @lds_wg_ld_seq_cst_single64(ptr addrspace(3) inreg %p) #1 {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v0, v0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_ld_seq_cst_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_read_b32 v0, v0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_ld_seq_cst_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_read_b32 v0, v0
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_read_b32 v0, v0
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_load_b32 v0, v0
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_ld_seq_cst_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_load_b32 v0, v0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_ld_seq_cst_single64:
; GFX1250: ; %bb.0:
@@ -2884,7 +3247,6 @@ define void @lds_wg_st_release_single64(ptr addrspace(3) inreg %p, i32 inreg %x)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_write_b32 v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
@@ -2897,7 +3259,6 @@ define void @lds_wg_st_release_single64(ptr addrspace(3) inreg %p, i32 inreg %x)
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: v_mov_b32_e32 v1, s1
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
@@ -2905,39 +3266,67 @@ define void @lds_wg_st_release_single64(ptr addrspace(3) inreg %p, i32 inreg %x)
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_st_release_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: v_mov_b32_e32 v1, s17
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_st_release_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s1
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_st_release_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_st_release_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_st_release_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_st_release_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_st_release_single64:
; GFX1250: ; %bb.0:
@@ -2964,9 +3353,7 @@ define void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) inreg %p) #1 {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 3
; GFX9-NEXT: v_mov_b32_e32 v0, s16
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_add_u32 v0, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
@@ -2978,52 +3365,78 @@ define void @lds_wg_rmw_add_acq_rel_single64(ptr addrspace(3) inreg %p) #1 {
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 3
; GFX942-NEXT: v_mov_b32_e32 v0, s0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_add_u32 v0, v1
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_rmw_add_acq_rel_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, 3
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_add_u32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_rmw_add_acq_rel_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, 3
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_add_u32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 3
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_add_u32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 3
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_add_u32 v0, v1
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 3
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_add_u32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_rmw_add_acq_rel_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 3
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_add_u32 v0, v1
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_rmw_add_acq_rel_single64:
; GFX1250: ; %bb.0:
@@ -3052,9 +3465,7 @@ define void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) inreg %p
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
@@ -3067,54 +3478,82 @@ define void @lds_wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(3) inreg %p
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: v_mov_b32_e32 v1, s1
; GFX942-NEXT: v_mov_b32_e32 v2, s2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: v_mov_b32_e32 v1, s17
-; GFX10-NEXT: v_mov_b32_e32 v2, s18
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v2, s1
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: ds_cmpstore_b32 v0, v1, v2
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-W32-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W64-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_storecnt 0x0
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W64-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_cmpxchg_acq_rel_monotonic_single64:
; GFX1250: ; %bb.0:
@@ -3215,7 +3654,6 @@ define void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) inreg %p
; GFX9-NEXT: v_mov_b32_e32 v1, s17
; GFX9-NEXT: v_mov_b32_e32 v2, s18
; GFX9-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v0, s16
; GFX9-NEXT: ds_write_b32 v0, v1
@@ -3229,46 +3667,75 @@ define void @lds_wg_cmpxchg_monotonic_acquire_single64(ptr addrspace(3) inreg %p
; GFX942-NEXT: v_mov_b32_e32 v1, s1
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NEXT: ds_write_b32 v0, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: v_mov_b32_e32 v1, s17
-; GFX10-NEXT: v_mov_b32_e32 v2, s18
-; GFX10-NEXT: ds_cmpst_b32 v0, v1, v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s16
-; GFX10-NEXT: ds_write_b32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v2, s1
-; GFX12-NEXT: ds_cmpstore_b32 v0, v1, v2
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ds_store_b32 v0, v1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W32-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W32-NEXT: ds_write_b32 v0, v1
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, s18
+; GFX10-W64-NEXT: ds_cmpst_b32 v0, v1, v2
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s16
+; GFX10-W64-NEXT: ds_write_b32 v0, v1
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W32-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: ds_store_b32 v0, v1
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, s1
+; GFX12-W64-NEXT: ds_cmpstore_b32 v0, v1, v2
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: ds_store_b32 v0, v1
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: lds_wg_cmpxchg_monotonic_acquire_single64:
; GFX1250: ; %bb.0:
@@ -3299,9 +3766,8 @@ define void @flat_wg_ld_acquire_single64(ptr addrspace(0) inreg %p) #1 {
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dword v0, v[0:1]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_store_dword v[0:1], v2
@@ -3315,56 +3781,95 @@ define void @flat_wg_ld_acquire_single64(ptr addrspace(0) inreg %p) #1 {
; GFX942-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GFX942-NEXT: s_mov_b32 s1, s2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-NEXT: flat_load_dword v0, v[0:1] sc0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: flat_load_dword v0, v[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: flat_store_dword v[0:1], v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: flat_wg_ld_acquire_single64:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s16
-; GFX10-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
-; GFX10-NEXT: s_mov_b32 s5, s17
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: flat_load_dword v0, v[0:1] glc
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: flat_store_dword v[0:1], v2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: flat_wg_ld_acquire_single64:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s2, s1
-; GFX12-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s1
-; GFX12-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SE
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: v_mov_b32_e32 v1, s1
-; GFX12-NEXT: flat_store_b32 v[0:1], v2
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX10-W32-LABEL: flat_wg_ld_acquire_single64:
+; GFX10-W32: ; %bb.0:
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: s_mov_b32 s4, s16
+; GFX10-W32-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W32-NEXT: s_mov_b32 s5, s17
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W32-NEXT: flat_load_dword v0, v[0:1] glc
+; GFX10-W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W32-NEXT: buffer_gl0_inv
+; GFX10-W32-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-W32-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W32-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W32-NEXT: flat_store_dword v[0:1], v2
+; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-W64-LABEL: flat_wg_ld_acquire_single64:
+; GFX10-W64: ; %bb.0:
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: s_mov_b32 s4, s16
+; GFX10-W64-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
+; GFX10-W64-NEXT: s_mov_b32 s5, s17
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W64-NEXT: flat_load_dword v0, v[0:1]
+; GFX10-W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-W64-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-W64-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-W64-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-W64-NEXT: flat_store_dword v[0:1], v2
+; GFX10-W64-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-W64-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W32-LABEL: flat_wg_ld_acquire_single64:
+; GFX12-W32: ; %bb.0:
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: s_wait_expcnt 0x0
+; GFX12-W32-NEXT: s_wait_samplecnt 0x0
+; GFX12-W32-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W32-NEXT: s_wait_kmcnt 0x0
+; GFX12-W32-NEXT: s_mov_b32 s2, s1
+; GFX12-W32-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: s_mov_b32 s1, s2
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W32-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SE
+; GFX12-W32-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W32-NEXT: global_inv scope:SCOPE_SE
+; GFX12-W32-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-W32-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W32-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-W32-NEXT: s_wait_dscnt 0x0
+; GFX12-W32-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-W64-LABEL: flat_wg_ld_acquire_single64:
+; GFX12-W64: ; %bb.0:
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: s_wait_expcnt 0x0
+; GFX12-W64-NEXT: s_wait_samplecnt 0x0
+; GFX12-W64-NEXT: s_wait_bvhcnt 0x0
+; GFX12-W64-NEXT: s_wait_kmcnt 0x0
+; GFX12-W64-NEXT: s_mov_b32 s2, s1
+; GFX12-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: s_mov_b32 s1, s2
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W64-NEXT: flat_load_b32 v0, v[0:1]
+; GFX12-W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-W64-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-W64-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-W64-NEXT: v_mov_b32_e32 v1, s1
+; GFX12-W64-NEXT: flat_store_b32 v[0:1], v2
+; GFX12-W64-NEXT: s_wait_dscnt 0x0
+; GFX12-W64-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: flat_wg_ld_acquire_single64:
; GFX1250: ; %bb.0:
@@ -3489,8 +3994,3 @@ define void @flat_wg_st_seq_cst_multi(ptr addrspace(0) inreg %p, i32 inreg %x) #
attributes #0 = { nounwind "amdgpu-flat-work-group-size"="32,32" "amdgpu-no-lds-dma" }
attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-lds-dma" }
attributes #2 = { nounwind "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-lds-dma" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10-W32: {{.*}}
-; GFX10-W64: {{.*}}
-; GFX12-W32: {{.*}}
-; GFX12-W64: {{.*}}
More information about the llvm-branch-commits
mailing list