[llvm-branch-commits] [llvm] [AMDGPU][NFC] Extract SIAtomicAddrSpace and fence MMRA parsing from SIMemoryLegalizer (PR #231854)
Fabian Ritter via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 15 05:53:02 PDT 2026
https://github.com/ritter-x2a updated https://github.com/llvm/llvm-project/pull/231854
>From a9246c0458c212a27c5503ef966478870e2b6cfc Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 14 Oct 2026 06:23:47 -0400
Subject: [PATCH 1/2] [AMDGPU][NFC] Extract SIAtomicAddrSpace and fence MMRA
parsing from SIMemoryLegalizer
Determining which memory accesses are ordered by a fence is already
useful in the middle-end; this PR extracts the code to do that from
SIMemoryLegalizer, so that it can be used in middle-end transformations.
So far, unexpected suffix values for the `amdgpu-synchronize-as` were
reported when they were encountered in SIMemoryLegalizer. This PR keeps
that and doesn't report them in the middle end, since that would lead to
a lot of repeated diagnostics for them.
Disclaimer: Claude gave review comments and suggestions that I addressed
during development.
Preparation for ROCM-29879.
Assisted-by: AI
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 65 ++++++++++++++
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 45 ++++++++++
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 93 +++-----------------
3 files changed, 120 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index 8a4887afd3e00a..837a5f7d7cc55c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -9,6 +9,7 @@
#include "AMDGPUMemoryUtils.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/MemorySSA.h"
@@ -17,6 +18,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/ReplaceConstant.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/AtomicOrdering.h"
@@ -478,4 +480,67 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
return false;
}
+SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) {
+ if (AS == AMDGPUAS::FLAT_ADDRESS)
+ return SIAtomicAddrSpace::FLAT;
+ if (AS == AMDGPUAS::GLOBAL_ADDRESS)
+ return SIAtomicAddrSpace::GLOBAL;
+ if (AS == AMDGPUAS::LOCAL_ADDRESS)
+ return SIAtomicAddrSpace::LDS;
+ if (AS == AMDGPUAS::PRIVATE_ADDRESS)
+ return SIAtomicAddrSpace::SCRATCH;
+ if (AS == AMDGPUAS::REGION_ADDRESS)
+ return SIAtomicAddrSpace::GDS;
+ if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
+ AS == AMDGPUAS::BUFFER_STRIDED_POINTER)
+ return SIAtomicAddrSpace::GLOBAL;
+
+ // Constant addrspaces map to OTHER because there is no point in atomic
+ // operations in them, since their memory does not change during kernel
+ // execution.
+ return SIAtomicAddrSpace::OTHER;
+}
+
+static const StringMap<SIAtomicAddrSpace> ASNames = {{
+ {"global", SIAtomicAddrSpace::GLOBAL},
+ {"local", SIAtomicAddrSpace::LDS},
+}};
+
+std::optional<SIAtomicAddrSpace>
+getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
+ function_ref<void(StringRef)> Diag) {
+ static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
+
+ if (!MMRA)
+ return std::nullopt;
+
+ SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE;
+ for (const auto &[Prefix, Suffix] : MMRA) {
+ if (Prefix != FenceASPrefix)
+ continue;
+
+ if (auto It = ASNames.find(Suffix); It != ASNames.end()) {
+ Result |= It->second;
+ } else {
+ SmallString<128> Str;
+ raw_svector_ostream OS(Str);
+ OS << "unknown address space '" << Suffix << "'; expected one of ";
+ ListSeparator LS;
+ for (const auto &[Name, Val] : ASNames)
+ OS << LS << '\'' << Name << '\'';
+ Diag(Str);
+ }
+ }
+
+ if (Result == SIAtomicAddrSpace::NONE)
+ return std::nullopt;
+
+ return Result;
+}
+
+std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I) {
+ const auto MMRA = MMRAMetadata(I);
+ return getSynchronizeAddrSpace(MMRA, [](StringRef) {});
+}
+
} // end namespace llvm::AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index 95f8c91a4dd939..e4bc43de91da80 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -10,9 +10,12 @@
#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
+#include <optional>
namespace llvm {
@@ -20,9 +23,11 @@ struct Align;
class AAResults;
class DataLayout;
class GlobalVariable;
+class Instruction;
class LoadInst;
class MemoryDef;
class MemorySSA;
+class MMRAMetadata;
class Value;
class Function;
class CallGraph;
@@ -113,6 +118,46 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA);
bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
AAResults *AA);
+/// The distinct address spaces supported by the AMDGPU target for
+/// atomic memory operation. Can be ORed together.
+enum class SIAtomicAddrSpace {
+ NONE = 0u,
+ GLOBAL = 1u << 0,
+ LDS = 1u << 1,
+ SCRATCH = 1u << 2,
+ GDS = 1u << 3,
+ OTHER = 1u << 4,
+
+ /// The address spaces that can be accessed by a FLAT instruction.
+ FLAT = GLOBAL | LDS | SCRATCH,
+
+ /// The address spaces that support atomic instructions.
+ ATOMIC = GLOBAL | LDS | SCRATCH | GDS,
+
+ /// All address spaces.
+ ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER,
+
+ LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
+};
+
+/// \return Return a bit set of the address spaces accessed by \p AS.
+SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS);
+
+/// Reads \p I's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
+/// If this tag isn't present, or if it has no meaningful values, returns
+/// \p none, otherwise returns the address spaces specified by the MD.
+/// Does not diagnose unexpected MMRA tag suffixes.
+std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I);
+
+/// Parses the values for the "amdgpu-synchronize-as" MMRA.
+/// If this tag isn't present, or if it has no meaningful values, returns
+/// \p none, otherwise returns the address spaces specified by the MD.
+/// If an unexpected suffix for the MMRA tag is found, \p Diag is called with
+/// an error message.
+std::optional<SIAtomicAddrSpace>
+getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
+ function_ref<void(StringRef)> Diag);
+
} // end namespace AMDGPU
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 10804d65b82b0f..518956fdeb36ca 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUMachineModuleInfo.h"
+#include "AMDGPUMemoryUtils.h"
#include "GCNSubtarget.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/StringExtras.h"
@@ -24,7 +25,6 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/PassManager.h"
-#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Debug.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
@@ -69,28 +69,6 @@ enum class SIAtomicScope {
SYSTEM
};
-/// The distinct address spaces supported by the AMDGPU target for
-/// atomic memory operation. Can be ORed together.
-enum class SIAtomicAddrSpace {
- NONE = 0u,
- GLOBAL = 1u << 0,
- LDS = 1u << 1,
- SCRATCH = 1u << 2,
- GDS = 1u << 3,
- OTHER = 1u << 4,
-
- /// The address spaces that can be accessed by a FLAT instruction.
- FLAT = GLOBAL | LDS | SCRATCH,
-
- /// The address spaces that support atomic instructions.
- ATOMIC = GLOBAL | LDS | SCRATCH | GDS,
-
- /// All address spaces.
- ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER,
-
- LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
-};
-
#ifndef NDEBUG
static StringRef toString(SIAtomicScope S) {
switch (S) {
@@ -292,9 +270,6 @@ class SIMemOpAccess final {
std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>>
toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrAddrSpace) const;
- /// \return Return a bit set of the address spaces accessed by \p AS.
- SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) const;
-
/// \returns Info constructed from \p MI, which has at least machine memory
/// operand.
std::optional<SIMemOpInfo>
@@ -687,50 +662,20 @@ class SIMemoryLegalizerLegacy final : public MachineFunctionPass {
bool runOnMachineFunction(MachineFunction &MF) override;
};
-static const StringMap<SIAtomicAddrSpace> ASNames = {{
- {"global", SIAtomicAddrSpace::GLOBAL},
- {"local", SIAtomicAddrSpace::LDS},
-}};
-
-void diagnoseUnknownMMRAASName(const MachineInstr &MI, StringRef AS) {
- const MachineFunction *MF = MI.getMF();
- const Function &Fn = MF->getFunction();
- SmallString<128> Str;
- raw_svector_ostream OS(Str);
- OS << "unknown address space '" << AS << "'; expected one of ";
- ListSeparator LS;
- for (const auto &[Name, Val] : ASNames)
- OS << LS << '\'' << Name << '\'';
- Fn.getContext().diagnose(
- DiagnosticInfoUnsupported(Fn, Str.str(), MI.getDebugLoc(), DS_Warning));
-}
-
/// Reads \p MI's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
/// If this tag isn't present, or if it has no meaningful values, returns
/// \p none, otherwise returns the address spaces specified by the MD.
+/// Issues a diagnostic if an unexpected suffix is observed for the MMRA tag.
static std::optional<SIAtomicAddrSpace>
getSynchronizeAddrSpaceMD(const MachineInstr &MI) {
- static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
-
- auto MMRA = MMRAMetadata(MI.getMMRAMetadata());
- if (!MMRA)
- return std::nullopt;
-
- SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE;
- for (const auto &[Prefix, Suffix] : MMRA) {
- if (Prefix != FenceASPrefix)
- continue;
-
- if (auto It = ASNames.find(Suffix); It != ASNames.end())
- Result |= It->second;
- else
- diagnoseUnknownMMRAASName(MI, Suffix);
- }
-
- if (Result == SIAtomicAddrSpace::NONE)
- return std::nullopt;
-
- return Result;
+ const auto MMRA = MMRAMetadata(MI.getMMRAMetadata());
+ auto Diag = [&MI](StringRef S) {
+ const MachineFunction *MF = MI.getMF();
+ const Function &Fn = MF->getFunction();
+ Fn.getContext().diagnose(
+ DiagnosticInfoUnsupported(Fn, S, MI.getDebugLoc(), DS_Warning));
+ };
+ return getSynchronizeAddrSpace(MMRA, Diag);
}
static void diagnoseUnknownAVMetadata(const MachineInstr &MI,
@@ -806,24 +751,6 @@ SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID,
return std::nullopt;
}
-SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const {
- if (AS == AMDGPUAS::FLAT_ADDRESS)
- return SIAtomicAddrSpace::FLAT;
- if (AS == AMDGPUAS::GLOBAL_ADDRESS)
- return SIAtomicAddrSpace::GLOBAL;
- if (AS == AMDGPUAS::LOCAL_ADDRESS)
- return SIAtomicAddrSpace::LDS;
- if (AS == AMDGPUAS::PRIVATE_ADDRESS)
- return SIAtomicAddrSpace::SCRATCH;
- if (AS == AMDGPUAS::REGION_ADDRESS)
- return SIAtomicAddrSpace::GDS;
- if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
- AS == AMDGPUAS::BUFFER_STRIDED_POINTER)
- return SIAtomicAddrSpace::GLOBAL;
-
- return SIAtomicAddrSpace::OTHER;
-}
-
SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_,
const GCNSubtarget &ST)
: MMI(&MMI_), ST(ST) {}
>From a15de6491e2ebc47ff0215336a3c4e8758ee245b Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Thu, 15 Oct 2026 08:47:17 -0400
Subject: [PATCH 2/2] Put the toSIAtomicAddrSpace logic into a table and make
it consistent with isExtendedGlobalAddrSpace
With that, the change is no longer NFC, since constant addrspaces and
addrspaces above `MAX_AMDGPU_ADDRESS` now map to `SIAtomicAddrSpace::GLOBAL`
instead of `OTHER`. This means that atomics on these addrspaces are no longer
rejected by SIMemoryLegalizer.
The tests only show differences in the generated ISA for volatile loads from
the constant addrspaces; so far we haven't waited directly after each such
volatile load, with this change we do. Arguably, these waits are unnecessary
since the accessed memory is not supposed to change, but, on the other hand,
one could argue that volatile accesses to memory that doesn't change are
pointless in the first place.
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 37 +++++-----
...licit-kernarg-backend-usage-global-isel.ll | 12 ++--
.../AMDGPU/GlobalISel/load-uniform-in-vgpr.ll | 18 ++---
...zero-and-sign-extending-uniform-in-vgpr.ll | 11 ++-
.../abi-attribute-hints-undefined-behavior.ll | 2 +
.../AMDGPU/agpr-copy-no-free-registers.ll | 4 +-
.../AMDGPU/implicit-kernarg-backend-usage.ll | 14 ++--
llvm/test/CodeGen/AMDGPU/load-hi16.ll | 8 ++-
.../memory-legalizer-invalid-addrspace.mir | 69 -------------------
.../AMDGPU/ptradd-sdag-optimizations.ll | 3 +
10 files changed, 60 insertions(+), 118 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index 837a5f7d7cc55c..c732c42a2cac70 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -480,24 +480,27 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
return false;
}
-SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) {
- if (AS == AMDGPUAS::FLAT_ADDRESS)
- return SIAtomicAddrSpace::FLAT;
- if (AS == AMDGPUAS::GLOBAL_ADDRESS)
+static constexpr SIAtomicAddrSpace LLVMToSIAtomicAddrSpaceMapping[] = {
+ SIAtomicAddrSpace::FLAT, //< AMDGPUAS::FLAT_ADDRESS
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::GLOBAL_ADDRESS
+ SIAtomicAddrSpace::GDS, //< AMDGPUAS::REGION_ADDRESS
+ SIAtomicAddrSpace::LDS, //< AMDGPUAS::LOCAL_ADDRESS
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS
+ SIAtomicAddrSpace::SCRATCH, //< AMDGPUAS::PRIVATE_ADDRESS
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS_32BIT
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_FAT_POINTER
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_RESOURCE
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_STRIDED_POINTER
+};
+
+SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned LLVMAddrSpace) {
+ constexpr unsigned SizeOfLLVMToSIAtomicAddrSpaceMapping =
+ sizeof(LLVMToSIAtomicAddrSpaceMapping) /
+ sizeof(LLVMToSIAtomicAddrSpaceMapping[0]);
+ if (LLVMAddrSpace > AMDGPUAS::MAX_AMDGPU_ADDRESS)
return SIAtomicAddrSpace::GLOBAL;
- if (AS == AMDGPUAS::LOCAL_ADDRESS)
- return SIAtomicAddrSpace::LDS;
- if (AS == AMDGPUAS::PRIVATE_ADDRESS)
- return SIAtomicAddrSpace::SCRATCH;
- if (AS == AMDGPUAS::REGION_ADDRESS)
- return SIAtomicAddrSpace::GDS;
- if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
- AS == AMDGPUAS::BUFFER_STRIDED_POINTER)
- return SIAtomicAddrSpace::GLOBAL;
-
- // Constant addrspaces map to OTHER because there is no point in atomic
- // operations in them, since their memory does not change during kernel
- // execution.
+ if (LLVMAddrSpace < SizeOfLLVMToSIAtomicAddrSpaceMapping)
+ return LLVMToSIAtomicAddrSpaceMapping[LLVMAddrSpace];
return SIAtomicAddrSpace::OTHER;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
index 110aee0d27646c..a50fc78fcf8075 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
@@ -310,8 +310,8 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V4-NEXT: v_mov_b32_e32 v0, s4
; GFX8V4-NEXT: v_mov_b32_e32 v1, s5
; GFX8V4-NEXT: flat_load_ubyte v0, v[0:1] glc
-; GFX8V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V4-NEXT: s_waitcnt vmcnt(0)
+; GFX8V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V4-NEXT: v_mov_b32_e32 v0, s10
; GFX8V4-NEXT: v_mov_b32_e32 v1, s11
; GFX8V4-NEXT: s_waitcnt lgkmcnt(0)
@@ -336,8 +336,8 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V5-NEXT: v_mov_b32_e32 v0, s4
; GFX8V5-NEXT: v_mov_b32_e32 v1, s5
; GFX8V5-NEXT: flat_load_ubyte v0, v[0:1] glc
-; GFX8V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V5-NEXT: s_waitcnt vmcnt(0)
+; GFX8V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V5-NEXT: v_mov_b32_e32 v0, s10
; GFX8V5-NEXT: v_mov_b32_e32 v1, s11
; GFX8V5-NEXT: s_waitcnt lgkmcnt(0)
@@ -351,10 +351,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V4: ; %bb.0:
; GFX9V4-NEXT: v_mov_b32_e32 v2, 0
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: s_waitcnt vmcnt(0)
+; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: v_mov_b32_e32 v0, s10
; GFX9V4-NEXT: v_mov_b32_e32 v1, s11
; GFX9V4-NEXT: ; kill: killed $sgpr6_sgpr7
@@ -368,10 +370,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V5: ; %bb.0:
; GFX9V5-NEXT: v_mov_b32_e32 v2, 0
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: s_waitcnt vmcnt(0)
+; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: v_mov_b32_e32 v0, s10
; GFX9V5-NEXT: v_mov_b32_e32 v1, s11
; GFX9V5-NEXT: ; kill: killed $sgpr6_sgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
index 730443a56b0033..095b0181001da7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
@@ -1237,10 +1237,9 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX7-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX7-NEXT: buffer_load_ushort v2, off, s[0:3], 0
; GFX7-NEXT: buffer_load_ushort v3, off, s[4:7], 0 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_readfirstlane_b32 s0, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s1, v3
; GFX7-NEXT: s_add_i32 s0, s0, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s0
@@ -1254,9 +1253,8 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX11-True16-NEXT: s_clause 0x1
; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[2:3] glc dlc
-; GFX11-True16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1270,9 +1268,8 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX11-NoTrue16-NEXT: s_clause 0x1
; GFX11-NoTrue16-NEXT: global_load_u16 v3, v2, s[0:1]
; GFX11-NoTrue16-NEXT: global_load_u16 v2, v2, s[2:3] glc dlc
-; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NoTrue16-NEXT: s_add_i32 s0, s0, s1
; GFX11-NoTrue16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1398,10 +1395,9 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_load_ushort v2, off, s[0:3], 0
; GFX7-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_readfirstlane_b32 s0, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s1, v3
; GFX7-NEXT: s_add_i32 s0, s0, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s0
@@ -1415,9 +1411,8 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX11-True16-NEXT: s_clause 0x1
; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] glc dlc
-; GFX11-True16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1431,9 +1426,8 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX11-NoTrue16-NEXT: s_clause 0x1
; GFX11-NoTrue16-NEXT: global_load_u16 v3, v2, s[0:1]
; GFX11-NoTrue16-NEXT: global_load_u16 v2, v2, s[0:1] glc dlc
-; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NoTrue16-NEXT: s_add_i32 s0, s0, s1
; GFX11-NoTrue16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll
index b57ebb2b5887f5..9a6dc3b706162d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll
@@ -479,12 +479,11 @@ define amdgpu_ps void @sextload_and_zextload_P4_i8_not_uniform_mmo_gfx12(ptr add
; GFX11-LABEL: sextload_and_zextload_P4_i8_not_uniform_mmo_gfx12:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_i8 v3, v2, s[0:1] glc dlc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_load_u8 v2, v2, s[2:3] glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -588,9 +587,8 @@ define amdgpu_ps void @sextload_P4_i16_not_natural_align_or_not_uniform_mmo_gfx1
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_i16 v3, v2, s[0:1]
; GFX11-NEXT: global_load_i16 v2, v2, s[2:3] glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -662,9 +660,8 @@ define amdgpu_ps void @zextload_P4_i16_not_natural_align_or_not_uniform_mmo_gfx1
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_u16 v3, v2, s[0:1]
; GFX11-NEXT: global_load_u16 v2, v2, s[2:3] glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
index a11cb4c4156172..9b1aed7b677255 100644
--- a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
+++ b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
@@ -256,6 +256,7 @@ define amdgpu_kernel void @marked_kernel_use_other_sgpr(ptr addrspace(1) %ptr) #
; FIXEDABI-NEXT: flat_load_ubyte v0, v[0:1] glc
; FIXEDABI-NEXT: s_waitcnt vmcnt(0)
; FIXEDABI-NEXT: flat_load_ubyte v0, v[0:1] glc
+; FIXEDABI-NEXT: s_waitcnt vmcnt(0)
; FIXEDABI-NEXT: s_endpgm
%queue.ptr = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
%implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
@@ -276,6 +277,7 @@ define amdgpu_kernel void @marked_kernel_nokernargs_implicitarg_ptr() #0 {
; FIXEDABI-NEXT: v_mov_b32_e32 v0, 0
; FIXEDABI-NEXT: v_mov_b32_e32 v1, 0
; FIXEDABI-NEXT: flat_load_ubyte v0, v[0:1] glc
+; FIXEDABI-NEXT: s_waitcnt vmcnt(0)
; FIXEDABI-NEXT: s_endpgm
%implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
%implicitarg.load = load volatile i8, ptr addrspace(4) %implicitarg.ptr
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index cb107810ac62ea..25418ec96de4b9 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -511,6 +511,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-LABEL: introduced_copy_to_sgpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: global_load_ushort v16, v[0:1], off glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
; GFX908-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x10
; GFX908-NEXT: s_load_dword s0, s[8:9], 0x18
@@ -547,7 +548,6 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: s_lshl_b64 s[14:15], s[10:11], 5
; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; GFX908-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s2, v16
; GFX908-NEXT: s_and_b32 s2, 0xffff, s2
; GFX908-NEXT: s_mul_i32 s3, s5, s2
@@ -655,6 +655,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-LABEL: introduced_copy_to_sgpr:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: global_load_ushort v18, v[0:1], off glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
; GFX90A-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x10
; GFX90A-NEXT: s_load_dword s0, s[8:9], 0x18
@@ -690,7 +691,6 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: s_lshl_b64 s[14:15], s[10:11], 5
; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; GFX90A-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s2, v18
; GFX90A-NEXT: s_and_b32 s2, 0xffff, s2
; GFX90A-NEXT: s_mul_i32 s3, s5, s2
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
index 387b9fc3b5f175..9a0c16826c600e 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
@@ -299,10 +299,11 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V4-NEXT: v_mov_b32_e32 v0, s4
; GFX8V4-NEXT: v_mov_b32_e32 v1, s5
; GFX8V4-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX8V4-NEXT: s_waitcnt vmcnt(0)
; GFX8V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V4-NEXT: v_mov_b32_e32 v2, s10
; GFX8V4-NEXT: v_mov_b32_e32 v3, s11
-; GFX8V4-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8V4-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V4-NEXT: v_mov_b32_e32 v0, s0
; GFX8V4-NEXT: v_mov_b32_e32 v1, s1
; GFX8V4-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
@@ -324,10 +325,11 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V5-NEXT: v_mov_b32_e32 v0, s4
; GFX8V5-NEXT: v_mov_b32_e32 v1, s5
; GFX8V5-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX8V5-NEXT: s_waitcnt vmcnt(0)
; GFX8V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V5-NEXT: v_mov_b32_e32 v2, s10
; GFX8V5-NEXT: v_mov_b32_e32 v3, s11
-; GFX8V5-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8V5-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V5-NEXT: v_mov_b32_e32 v0, s0
; GFX8V5-NEXT: v_mov_b32_e32 v1, s1
; GFX8V5-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
@@ -338,10 +340,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V4: ; %bb.0:
; GFX9V4-NEXT: v_mov_b32_e32 v2, 0
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: s_waitcnt vmcnt(0)
+; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: v_mov_b32_e32 v0, s10
; GFX9V4-NEXT: v_mov_b32_e32 v1, s11
; GFX9V4-NEXT: ; kill: killed $sgpr6_sgpr7
@@ -355,10 +359,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V5: ; %bb.0:
; GFX9V5-NEXT: v_mov_b32_e32 v2, 0
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: s_waitcnt vmcnt(0)
+; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: v_mov_b32_e32 v0, s10
; GFX9V5-NEXT: v_mov_b32_e32 v1, s11
; GFX9V5-NEXT: ; kill: killed $sgpr6_sgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/load-hi16.ll b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
index cd8bc4f9c202b2..cf0c795d95e0e9 100644
--- a/llvm/test/CodeGen/AMDGPU/load-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
@@ -2549,7 +2549,7 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX900: ; %bb.0: ; %entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: global_load_ushort v2, v[0:1], off glc
-; GFX900-NEXT: s_nop 0
+; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:2 glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v0, v2
@@ -2559,9 +2559,10 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX906: ; %bb.0: ; %entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-NEXT: global_load_ushort v2, v[0:1], off glc
+; GFX906-NEXT: s_waitcnt vmcnt(0)
; GFX906-NEXT: global_load_ushort v3, v[0:1], off offset:2 glc
-; GFX906-NEXT: s_mov_b32 s4, 0x5040100
; GFX906-NEXT: s_waitcnt vmcnt(0)
+; GFX906-NEXT: s_mov_b32 s4, 0x5040100
; GFX906-NEXT: v_perm_b32 v0, v3, v2, s4
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
@@ -2571,6 +2572,7 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX803-NEXT: v_add_u32_e32 v2, vcc, 2, v0
; GFX803-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; GFX803-NEXT: flat_load_ushort v0, v[0:1] glc
+; GFX803-NEXT: s_waitcnt vmcnt(0)
; GFX803-NEXT: flat_load_ushort v1, v[2:3] glc
; GFX803-NEXT: s_waitcnt vmcnt(0)
; GFX803-NEXT: v_lshlrev_b32_e32 v1, 16, v1
@@ -2581,7 +2583,7 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX900-FLATSCR: ; %bb.0: ; %entry
; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-FLATSCR-NEXT: global_load_ushort v2, v[0:1], off glc
-; GFX900-FLATSCR-NEXT: s_nop 0
+; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX900-FLATSCR-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:2 glc
; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX900-FLATSCR-NEXT: v_mov_b32_e32 v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir
deleted file mode 100644
index 63ee919fe1386d..00000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir
+++ /dev/null
@@ -1,69 +0,0 @@
-# RUN: not llc -mtriple=amdgpu8.03 -run-pass si-memory-legalizer %s -o - 2>&1 | FileCheck -check-prefix=GCN %s
-
----
-
-# GCN: error: <unknown>:0:0: in function invalid_load void (): Unsupported atomic address space
-
-name: invalid_load
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr0 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr2_sgpr3
- $vgpr1 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $sgpr2_sgpr3, implicit $exec
- renamable $vgpr2 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load syncscope("one-as") seq_cst (s32) from `ptr addrspace(42) poison`)
- $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
- S_ENDPGM 0
-
-...
----
-
-# GCN: error: <unknown>:0:0: in function invalid_store void (): Unsupported atomic address space
-
-name: invalid_store
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (volatile store syncscope("agent-one-as") seq_cst (s32) into `ptr addrspace(42) poison`)
- S_ENDPGM 0
-
-...
----
-
-# GCN: error: <unknown>:0:0: in function invalid_cmpxchg void (): Unsupported atomic address space
-
-name: invalid_cmpxchg
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr3 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit-def $vgpr2_vgpr3, implicit $sgpr0_sgpr1
- $vgpr0 = V_MOV_B32_e32 killed $sgpr4, implicit $exec, implicit $exec
- $vgpr1 = V_MOV_B32_e32 killed $sgpr5, implicit $exec, implicit $exec
- $vgpr2 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- FLAT_ATOMIC_CMPSWAP killed renamable $vgpr2_vgpr3, killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load store syncscope("workgroup-one-as") seq_cst seq_cst (s32) on `ptr addrspace(42) poison`)
- S_ENDPGM 0
-
-...
----
-
-# GCN: error: <unknown>:0:0: in function invalid_rmw void (): Unsupported atomic address space
-
-name: invalid_rmw
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr0 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr2_sgpr3
- $vgpr1 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $sgpr2_sgpr3, implicit $exec
- $vgpr2 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- FLAT_ATOMIC_SWAP killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load store syncscope("wavefront-one-as") seq_cst (s32) on `ptr addrspace(42) poison`)
- S_ENDPGM 0
-
-...
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 20af74faa5c193..f8567bc3b0c5af 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -96,8 +96,11 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_ubyte v1, v0, s[2:3] sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_load_ubyte v1, v0, s[4:5] offset:8 sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_load_ubyte v1, v0, s[0:1] sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: ; kill: killed $sgpr0_sgpr1
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
More information about the llvm-branch-commits
mailing list