[llvm-branch-commits] [llvm] [AMDGPU] Extract SIAtomicAddrSpace and fence MMRA parsing from SIMemoryLegalizer (PR #231854)
Fabian Ritter via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 00:56:47 PST 2026
https://github.com/ritter-x2a updated https://github.com/llvm/llvm-project/pull/231854
>From 1d1fecbac436b660a5fc3a5d0734076cf07f4cb3 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 14 Oct 2026 06:23:47 -0400
Subject: [PATCH 1/5] [AMDGPU][NFC] Extract SIAtomicAddrSpace and fence MMRA
parsing from SIMemoryLegalizer
Determining which memory accesses are ordered by a fence is already
useful in the middle-end; this PR extracts the code to do that from
SIMemoryLegalizer, so that it can be used in middle-end transformations.
So far, unexpected suffix values for the `amdgpu-synchronize-as` were
reported when they were encountered in SIMemoryLegalizer. This PR keeps
that and doesn't report them in the middle end, since that would lead to
a lot of repeated diagnostics for them.
Disclaimer: Claude gave review comments and suggestions that I addressed
during development.
Preparation for ROCM-29879.
Assisted-by: AI
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 65 ++++++++++++++
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 45 ++++++++++
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 93 +++-----------------
3 files changed, 120 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index c495e2a8d46f22..a914ab2a8f66a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -9,6 +9,7 @@
#include "AMDGPUMemoryUtils.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/MemorySSA.h"
@@ -17,6 +18,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/ReplaceConstant.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/AtomicOrdering.h"
@@ -478,4 +480,67 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
return false;
}
+SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) {
+ if (AS == AMDGPUAS::FLAT_ADDRESS)
+ return SIAtomicAddrSpace::FLAT;
+ if (AS == AMDGPUAS::GLOBAL_ADDRESS)
+ return SIAtomicAddrSpace::GLOBAL;
+ if (AS == AMDGPUAS::LOCAL_ADDRESS)
+ return SIAtomicAddrSpace::LDS;
+ if (AS == AMDGPUAS::PRIVATE_ADDRESS)
+ return SIAtomicAddrSpace::SCRATCH;
+ if (AS == AMDGPUAS::REGION_ADDRESS)
+ return SIAtomicAddrSpace::GDS;
+ if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
+ AS == AMDGPUAS::BUFFER_STRIDED_POINTER)
+ return SIAtomicAddrSpace::GLOBAL;
+
+ // Constant addrspaces map to OTHER because there is no point in atomic
+ // operations in them, since their memory does not change during kernel
+ // execution.
+ return SIAtomicAddrSpace::OTHER;
+}
+
+static const StringMap<SIAtomicAddrSpace> ASNames = {{
+ {"global", SIAtomicAddrSpace::GLOBAL},
+ {"local", SIAtomicAddrSpace::LDS},
+}};
+
+std::optional<SIAtomicAddrSpace>
+getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
+ function_ref<void(StringRef)> Diag) {
+ static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
+
+ if (!MMRA)
+ return std::nullopt;
+
+ SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE;
+ for (const auto &[Prefix, Suffix] : MMRA) {
+ if (Prefix != FenceASPrefix)
+ continue;
+
+ if (auto It = ASNames.find(Suffix); It != ASNames.end()) {
+ Result |= It->second;
+ } else {
+ SmallString<128> Str;
+ raw_svector_ostream OS(Str);
+ OS << "unknown address space '" << Suffix << "'; expected one of ";
+ ListSeparator LS;
+ for (const auto &[Name, Val] : ASNames)
+ OS << LS << '\'' << Name << '\'';
+ Diag(Str);
+ }
+ }
+
+ if (Result == SIAtomicAddrSpace::NONE)
+ return std::nullopt;
+
+ return Result;
+}
+
+std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I) {
+ const auto MMRA = MMRAMetadata(I);
+ return getSynchronizeAddrSpace(MMRA, [](StringRef) {});
+}
+
} // end namespace llvm::AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index 95f8c91a4dd939..e4bc43de91da80 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -10,9 +10,12 @@
#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUMEMORYUTILS_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
+#include <optional>
namespace llvm {
@@ -20,9 +23,11 @@ struct Align;
class AAResults;
class DataLayout;
class GlobalVariable;
+class Instruction;
class LoadInst;
class MemoryDef;
class MemorySSA;
+class MMRAMetadata;
class Value;
class Function;
class CallGraph;
@@ -113,6 +118,46 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA);
bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
AAResults *AA);
+/// The distinct address spaces supported by the AMDGPU target for
+/// atomic memory operation. Can be ORed together.
+enum class SIAtomicAddrSpace {
+ NONE = 0u,
+ GLOBAL = 1u << 0,
+ LDS = 1u << 1,
+ SCRATCH = 1u << 2,
+ GDS = 1u << 3,
+ OTHER = 1u << 4,
+
+ /// The address spaces that can be accessed by a FLAT instruction.
+ FLAT = GLOBAL | LDS | SCRATCH,
+
+ /// The address spaces that support atomic instructions.
+ ATOMIC = GLOBAL | LDS | SCRATCH | GDS,
+
+ /// All address spaces.
+ ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER,
+
+ LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
+};
+
+/// \return Return a bit set of the address spaces accessed by \p AS.
+SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS);
+
+/// Reads \p I's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
+/// If this tag isn't present, or if it has no meaningful values, returns
+/// \p none, otherwise returns the address spaces specified by the MD.
+/// Does not diagnose unexpected MMRA tag suffixes.
+std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I);
+
+/// Parses the values for the "amdgpu-synchronize-as" MMRA.
+/// If this tag isn't present, or if it has no meaningful values, returns
+/// \p none, otherwise returns the address spaces specified by the MD.
+/// If an unexpected suffix for the MMRA tag is found, \p Diag is called with
+/// an error message.
+std::optional<SIAtomicAddrSpace>
+getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
+ function_ref<void(StringRef)> Diag);
+
} // end namespace AMDGPU
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 01d01ccbfc546c..dd84db28c66a06 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -15,6 +15,7 @@
#include "AMDGPU.h"
#include "AMDGPUMachineModuleInfo.h"
+#include "AMDGPUMemoryUtils.h"
#include "GCNSubtarget.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/StringExtras.h"
@@ -24,7 +25,6 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
#include "llvm/IR/PassManager.h"
-#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Debug.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
@@ -65,28 +65,6 @@ enum class SIAtomicScope {
SYSTEM
};
-/// The distinct address spaces supported by the AMDGPU target for
-/// atomic memory operation. Can be ORed together.
-enum class SIAtomicAddrSpace {
- NONE = 0u,
- GLOBAL = 1u << 0,
- LDS = 1u << 1,
- SCRATCH = 1u << 2,
- GDS = 1u << 3,
- OTHER = 1u << 4,
-
- /// The address spaces that can be accessed by a FLAT instruction.
- FLAT = GLOBAL | LDS | SCRATCH,
-
- /// The address spaces that support atomic instructions.
- ATOMIC = GLOBAL | LDS | SCRATCH | GDS,
-
- /// All address spaces.
- ALL = GLOBAL | LDS | SCRATCH | GDS | OTHER,
-
- LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
-};
-
#ifndef NDEBUG
static StringRef toString(SIAtomicScope S) {
switch (S) {
@@ -293,9 +271,6 @@ class SIMemOpAccess final {
std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>>
toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrAddrSpace) const;
- /// \return Return a bit set of the address spaces accessed by \p AS.
- SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) const;
-
/// \returns Info constructed from \p MI, which has at least machine memory
/// operand.
std::optional<SIMemOpInfo>
@@ -697,50 +672,20 @@ class SIMemoryLegalizerLegacy final : public MachineFunctionPass {
bool runOnMachineFunction(MachineFunction &MF) override;
};
-static const StringMap<SIAtomicAddrSpace> ASNames = {{
- {"global", SIAtomicAddrSpace::GLOBAL},
- {"local", SIAtomicAddrSpace::LDS},
-}};
-
-void diagnoseUnknownMMRAASName(const MachineInstr &MI, StringRef AS) {
- const MachineFunction *MF = MI.getMF();
- const Function &Fn = MF->getFunction();
- SmallString<128> Str;
- raw_svector_ostream OS(Str);
- OS << "unknown address space '" << AS << "'; expected one of ";
- ListSeparator LS;
- for (const auto &[Name, Val] : ASNames)
- OS << LS << '\'' << Name << '\'';
- Fn.getContext().diagnose(
- DiagnosticInfoUnsupported(Fn, Str.str(), MI.getDebugLoc(), DS_Warning));
-}
-
/// Reads \p MI's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
/// If this tag isn't present, or if it has no meaningful values, returns
/// \p none, otherwise returns the address spaces specified by the MD.
+/// Issues a diagnostic if an unexpected suffix is observed for the MMRA tag.
static std::optional<SIAtomicAddrSpace>
getSynchronizeAddrSpaceMD(const MachineInstr &MI) {
- static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
-
- auto MMRA = MMRAMetadata(MI.getMMRAMetadata());
- if (!MMRA)
- return std::nullopt;
-
- SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE;
- for (const auto &[Prefix, Suffix] : MMRA) {
- if (Prefix != FenceASPrefix)
- continue;
-
- if (auto It = ASNames.find(Suffix); It != ASNames.end())
- Result |= It->second;
- else
- diagnoseUnknownMMRAASName(MI, Suffix);
- }
-
- if (!any(Result))
- return std::nullopt;
-
- return Result;
+ const auto MMRA = MMRAMetadata(MI.getMMRAMetadata());
+ auto Diag = [&MI](StringRef S) {
+ const MachineFunction *MF = MI.getMF();
+ const Function &Fn = MF->getFunction();
+ Fn.getContext().diagnose(
+ DiagnosticInfoUnsupported(Fn, S, MI.getDebugLoc(), DS_Warning));
+ };
+ return getSynchronizeAddrSpace(MMRA, Diag);
}
static void diagnoseUnknownAVMetadata(const MachineInstr &MI,
@@ -816,24 +761,6 @@ SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID,
return std::nullopt;
}
-SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const {
- if (AS == AMDGPUAS::FLAT_ADDRESS)
- return SIAtomicAddrSpace::FLAT;
- if (AS == AMDGPUAS::GLOBAL_ADDRESS)
- return SIAtomicAddrSpace::GLOBAL;
- if (AS == AMDGPUAS::LOCAL_ADDRESS)
- return SIAtomicAddrSpace::LDS;
- if (AS == AMDGPUAS::PRIVATE_ADDRESS)
- return SIAtomicAddrSpace::SCRATCH;
- if (AS == AMDGPUAS::REGION_ADDRESS)
- return SIAtomicAddrSpace::GDS;
- if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
- AS == AMDGPUAS::BUFFER_STRIDED_POINTER)
- return SIAtomicAddrSpace::GLOBAL;
-
- return SIAtomicAddrSpace::OTHER;
-}
-
// TODO: Consider moving single-wave workgroup->wavefront scope relaxation to an
// IR pass (and extending it to other scoped operations), so middle-end
// optimizations see wavefront scope earlier.
>From 7aca8cdaca996c87d64e154ff834ada0b8ee2e65 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Thu, 15 Oct 2026 08:47:17 -0400
Subject: [PATCH 2/5] Put the toSIAtomicAddrSpace logic into a table and make
it consistent with isExtendedGlobalAddrSpace
With that, the change is no longer NFC, since constant addrspaces and
addrspaces above `MAX_AMDGPU_ADDRESS` now map to `SIAtomicAddrSpace::GLOBAL`
instead of `OTHER`. This means that atomics on these addrspaces are no longer
rejected by SIMemoryLegalizer.
The tests only show differences in the generated ISA for volatile loads from
the constant addrspaces; so far we haven't waited directly after each such
volatile load, with this change we do. Arguably, these waits are unnecessary
since the accessed memory is not supposed to change, but, on the other hand,
one could argue that volatile accesses to memory that doesn't change are
pointless in the first place.
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 37 +++++-----
...licit-kernarg-backend-usage-global-isel.ll | 12 ++-
.../AMDGPU/GlobalISel/load-uniform-in-vgpr.ll | 18 ++---
...zero-and-sign-extending-uniform-in-vgpr.ll | 11 +--
.../abi-attribute-hints-undefined-behavior.ll | 2 +
.../AMDGPU/agpr-copy-no-free-registers.ll | 4 +-
.../AMDGPU/implicit-kernarg-backend-usage.ll | 14 +++-
llvm/test/CodeGen/AMDGPU/load-hi16.ll | 8 +-
.../memory-legalizer-invalid-addrspace.mir | 73 -------------------
.../AMDGPU/ptradd-sdag-optimizations.ll | 3 +
10 files changed, 60 insertions(+), 122 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index a914ab2a8f66a2..d1583575afe422 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -480,24 +480,27 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
return false;
}
-SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS) {
- if (AS == AMDGPUAS::FLAT_ADDRESS)
- return SIAtomicAddrSpace::FLAT;
- if (AS == AMDGPUAS::GLOBAL_ADDRESS)
+static constexpr SIAtomicAddrSpace LLVMToSIAtomicAddrSpaceMapping[] = {
+ SIAtomicAddrSpace::FLAT, //< AMDGPUAS::FLAT_ADDRESS
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::GLOBAL_ADDRESS
+ SIAtomicAddrSpace::GDS, //< AMDGPUAS::REGION_ADDRESS
+ SIAtomicAddrSpace::LDS, //< AMDGPUAS::LOCAL_ADDRESS
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS
+ SIAtomicAddrSpace::SCRATCH, //< AMDGPUAS::PRIVATE_ADDRESS
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS_32BIT
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_FAT_POINTER
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_RESOURCE
+ SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_STRIDED_POINTER
+};
+
+SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned LLVMAddrSpace) {
+ constexpr unsigned SizeOfLLVMToSIAtomicAddrSpaceMapping =
+ sizeof(LLVMToSIAtomicAddrSpaceMapping) /
+ sizeof(LLVMToSIAtomicAddrSpaceMapping[0]);
+ if (LLVMAddrSpace > AMDGPUAS::MAX_AMDGPU_ADDRESS)
return SIAtomicAddrSpace::GLOBAL;
- if (AS == AMDGPUAS::LOCAL_ADDRESS)
- return SIAtomicAddrSpace::LDS;
- if (AS == AMDGPUAS::PRIVATE_ADDRESS)
- return SIAtomicAddrSpace::SCRATCH;
- if (AS == AMDGPUAS::REGION_ADDRESS)
- return SIAtomicAddrSpace::GDS;
- if (AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
- AS == AMDGPUAS::BUFFER_STRIDED_POINTER)
- return SIAtomicAddrSpace::GLOBAL;
-
- // Constant addrspaces map to OTHER because there is no point in atomic
- // operations in them, since their memory does not change during kernel
- // execution.
+ if (LLVMAddrSpace < SizeOfLLVMToSIAtomicAddrSpaceMapping)
+ return LLVMToSIAtomicAddrSpaceMapping[LLVMAddrSpace];
return SIAtomicAddrSpace::OTHER;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
index 110aee0d27646c..a50fc78fcf8075 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll
@@ -310,8 +310,8 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V4-NEXT: v_mov_b32_e32 v0, s4
; GFX8V4-NEXT: v_mov_b32_e32 v1, s5
; GFX8V4-NEXT: flat_load_ubyte v0, v[0:1] glc
-; GFX8V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V4-NEXT: s_waitcnt vmcnt(0)
+; GFX8V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V4-NEXT: v_mov_b32_e32 v0, s10
; GFX8V4-NEXT: v_mov_b32_e32 v1, s11
; GFX8V4-NEXT: s_waitcnt lgkmcnt(0)
@@ -336,8 +336,8 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V5-NEXT: v_mov_b32_e32 v0, s4
; GFX8V5-NEXT: v_mov_b32_e32 v1, s5
; GFX8V5-NEXT: flat_load_ubyte v0, v[0:1] glc
-; GFX8V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V5-NEXT: s_waitcnt vmcnt(0)
+; GFX8V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V5-NEXT: v_mov_b32_e32 v0, s10
; GFX8V5-NEXT: v_mov_b32_e32 v1, s11
; GFX8V5-NEXT: s_waitcnt lgkmcnt(0)
@@ -351,10 +351,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V4: ; %bb.0:
; GFX9V4-NEXT: v_mov_b32_e32 v2, 0
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: s_waitcnt vmcnt(0)
+; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: v_mov_b32_e32 v0, s10
; GFX9V4-NEXT: v_mov_b32_e32 v1, s11
; GFX9V4-NEXT: ; kill: killed $sgpr6_sgpr7
@@ -368,10 +370,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V5: ; %bb.0:
; GFX9V5-NEXT: v_mov_b32_e32 v2, 0
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: s_waitcnt vmcnt(0)
+; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: v_mov_b32_e32 v0, s10
; GFX9V5-NEXT: v_mov_b32_e32 v1, s11
; GFX9V5-NEXT: ; kill: killed $sgpr6_sgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
index 730443a56b0033..095b0181001da7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
@@ -1237,10 +1237,9 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX7-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX7-NEXT: buffer_load_ushort v2, off, s[0:3], 0
; GFX7-NEXT: buffer_load_ushort v3, off, s[4:7], 0 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_readfirstlane_b32 s0, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s1, v3
; GFX7-NEXT: s_add_i32 s0, s0, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s0
@@ -1254,9 +1253,8 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX11-True16-NEXT: s_clause 0x1
; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[2:3] glc dlc
-; GFX11-True16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1270,9 +1268,8 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX11-NoTrue16-NEXT: s_clause 0x1
; GFX11-NoTrue16-NEXT: global_load_u16 v3, v2, s[0:1]
; GFX11-NoTrue16-NEXT: global_load_u16 v2, v2, s[2:3] glc dlc
-; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NoTrue16-NEXT: s_add_i32 s0, s0, s1
; GFX11-NoTrue16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1398,10 +1395,9 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_load_ushort v2, off, s[0:3], 0
; GFX7-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc
+; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_waitcnt vmcnt(1)
; GFX7-NEXT: v_readfirstlane_b32 s0, v2
-; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_readfirstlane_b32 s1, v3
; GFX7-NEXT: s_add_i32 s0, s0, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s0
@@ -1415,9 +1411,8 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX11-True16-NEXT: s_clause 0x1
; GFX11-True16-NEXT: global_load_d16_b16 v3, v2, s[0:1]
; GFX11-True16-NEXT: global_load_d16_b16 v2, v2, s[0:1] glc dlc
-; GFX11-True16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-True16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1431,9 +1426,8 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX11-NoTrue16-NEXT: s_clause 0x1
; GFX11-NoTrue16-NEXT: global_load_u16 v3, v2, s[0:1]
; GFX11-NoTrue16-NEXT: global_load_u16 v2, v2, s[0:1] glc dlc
-; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NoTrue16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NoTrue16-NEXT: s_add_i32 s0, s0, s1
; GFX11-NoTrue16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll
index b57ebb2b5887f5..9a6dc3b706162d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-zero-and-sign-extending-uniform-in-vgpr.ll
@@ -479,12 +479,11 @@ define amdgpu_ps void @sextload_and_zextload_P4_i8_not_uniform_mmo_gfx12(ptr add
; GFX11-LABEL: sextload_and_zextload_P4_i8_not_uniform_mmo_gfx12:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_i8 v3, v2, s[0:1] glc dlc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: global_load_u8 v2, v2, s[2:3] glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -588,9 +587,8 @@ define amdgpu_ps void @sextload_P4_i16_not_natural_align_or_not_uniform_mmo_gfx1
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_i16 v3, v2, s[0:1]
; GFX11-NEXT: global_load_i16 v2, v2, s[2:3] glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -662,9 +660,8 @@ define amdgpu_ps void @zextload_P4_i16_not_natural_align_or_not_uniform_mmo_gfx1
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_u16 v3, v2, s[0:1]
; GFX11-NEXT: global_load_u16 v2, v2, s[2:3] glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v3
; GFX11-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-NEXT: s_add_i32 s0, s0, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
index a11cb4c4156172..9b1aed7b677255 100644
--- a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
+++ b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll
@@ -256,6 +256,7 @@ define amdgpu_kernel void @marked_kernel_use_other_sgpr(ptr addrspace(1) %ptr) #
; FIXEDABI-NEXT: flat_load_ubyte v0, v[0:1] glc
; FIXEDABI-NEXT: s_waitcnt vmcnt(0)
; FIXEDABI-NEXT: flat_load_ubyte v0, v[0:1] glc
+; FIXEDABI-NEXT: s_waitcnt vmcnt(0)
; FIXEDABI-NEXT: s_endpgm
%queue.ptr = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
%implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
@@ -276,6 +277,7 @@ define amdgpu_kernel void @marked_kernel_nokernargs_implicitarg_ptr() #0 {
; FIXEDABI-NEXT: v_mov_b32_e32 v0, 0
; FIXEDABI-NEXT: v_mov_b32_e32 v1, 0
; FIXEDABI-NEXT: flat_load_ubyte v0, v[0:1] glc
+; FIXEDABI-NEXT: s_waitcnt vmcnt(0)
; FIXEDABI-NEXT: s_endpgm
%implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
%implicitarg.load = load volatile i8, ptr addrspace(4) %implicitarg.ptr
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 81e82618180355..b20ab3beedbbb4 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -511,6 +511,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-LABEL: introduced_copy_to_sgpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: global_load_ushort v16, v[0:1], off glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_load_dwordx4 s[20:23], s[8:9], 0x0
; GFX908-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x10
; GFX908-NEXT: s_load_dword s0, s[8:9], 0x18
@@ -547,7 +548,6 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: s_lshl_b64 s[14:15], s[6:7], 5
; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; GFX908-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
-; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s2, v16
; GFX908-NEXT: s_and_b32 s2, 0xffff, s2
; GFX908-NEXT: s_mul_i32 s3, s21, s2
@@ -655,6 +655,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-LABEL: introduced_copy_to_sgpr:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: global_load_ushort v18, v[0:1], off glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_load_dwordx4 s[20:23], s[8:9], 0x0
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x10
; GFX90A-NEXT: s_load_dword s0, s[8:9], 0x18
@@ -690,7 +691,6 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: s_lshl_b64 s[14:15], s[6:7], 5
; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; GFX90A-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s2, v18
; GFX90A-NEXT: s_and_b32 s2, 0xffff, s2
; GFX90A-NEXT: s_mul_i32 s3, s21, s2
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
index 387b9fc3b5f175..9a0c16826c600e 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
@@ -299,10 +299,11 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V4-NEXT: v_mov_b32_e32 v0, s4
; GFX8V4-NEXT: v_mov_b32_e32 v1, s5
; GFX8V4-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX8V4-NEXT: s_waitcnt vmcnt(0)
; GFX8V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V4-NEXT: v_mov_b32_e32 v2, s10
; GFX8V4-NEXT: v_mov_b32_e32 v3, s11
-; GFX8V4-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8V4-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V4-NEXT: v_mov_b32_e32 v0, s0
; GFX8V4-NEXT: v_mov_b32_e32 v1, s1
; GFX8V4-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
@@ -324,10 +325,11 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX8V5-NEXT: v_mov_b32_e32 v0, s4
; GFX8V5-NEXT: v_mov_b32_e32 v1, s5
; GFX8V5-NEXT: flat_load_ubyte v0, v[0:1] glc
+; GFX8V5-NEXT: s_waitcnt vmcnt(0)
; GFX8V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8V5-NEXT: v_mov_b32_e32 v2, s10
; GFX8V5-NEXT: v_mov_b32_e32 v3, s11
-; GFX8V5-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8V5-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V5-NEXT: v_mov_b32_e32 v0, s0
; GFX8V5-NEXT: v_mov_b32_e32 v1, s1
; GFX8V5-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
@@ -338,10 +340,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V4: ; %bb.0:
; GFX9V4-NEXT: v_mov_b32_e32 v2, 0
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: s_waitcnt vmcnt(0)
+; GFX9V4-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V4-NEXT: v_mov_b32_e32 v0, s10
; GFX9V4-NEXT: v_mov_b32_e32 v1, s11
; GFX9V4-NEXT: ; kill: killed $sgpr6_sgpr7
@@ -355,10 +359,12 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) #0 {
; GFX9V5: ; %bb.0:
; GFX9V5-NEXT: v_mov_b32_e32 v2, 0
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[6:7] glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[8:9] offset:8 glc
+; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: global_load_ubyte v0, v2, s[4:5] glc
-; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: s_waitcnt vmcnt(0)
+; GFX9V5-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9V5-NEXT: v_mov_b32_e32 v0, s10
; GFX9V5-NEXT: v_mov_b32_e32 v1, s11
; GFX9V5-NEXT: ; kill: killed $sgpr6_sgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/load-hi16.ll b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
index 4268e5cdda690f..67ea5d7a0aaef7 100644
--- a/llvm/test/CodeGen/AMDGPU/load-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
@@ -2549,7 +2549,7 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX900: ; %bb.0: ; %entry
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: global_load_ushort v2, v[0:1], off glc
-; GFX900-NEXT: s_nop 0
+; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:2 glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v0, v2
@@ -2559,9 +2559,10 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX906: ; %bb.0: ; %entry
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX906-NEXT: global_load_ushort v2, v[0:1], off glc
+; GFX906-NEXT: s_waitcnt vmcnt(0)
; GFX906-NEXT: global_load_ushort v3, v[0:1], off offset:2 glc
-; GFX906-NEXT: s_mov_b32 s4, 0x5040100
; GFX906-NEXT: s_waitcnt vmcnt(0)
+; GFX906-NEXT: s_mov_b32 s4, 0x5040100
; GFX906-NEXT: v_perm_b32 v0, v3, v2, s4
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
@@ -2571,6 +2572,7 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX803-NEXT: v_add_u32_e32 v2, vcc, 2, v0
; GFX803-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; GFX803-NEXT: flat_load_ushort v0, v[0:1] glc
+; GFX803-NEXT: s_waitcnt vmcnt(0)
; GFX803-NEXT: flat_load_ushort v1, v[2:3] glc
; GFX803-NEXT: s_waitcnt vmcnt(0)
; GFX803-NEXT: v_lshlrev_b32_e32 v1, 16, v1
@@ -2581,7 +2583,7 @@ define <2 x i16> @load_constant_v2i16_split(ptr addrspace(4) %in) #0 {
; GFX900-FLATSCR: ; %bb.0: ; %entry
; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-FLATSCR-NEXT: global_load_ushort v2, v[0:1], off glc
-; GFX900-FLATSCR-NEXT: s_nop 0
+; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX900-FLATSCR-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:2 glc
; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX900-FLATSCR-NEXT: v_mov_b32_e32 v0, v2
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir
deleted file mode 100644
index 0ae727c5fda2ed..00000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-invalid-addrspace.mir
+++ /dev/null
@@ -1,73 +0,0 @@
-# RUN: not llc -mtriple=amdgpu8.03 -run-pass si-memory-legalizer %s -o - 2>&1 | FileCheck -check-prefix=GCN %s
-
----
-
-# GCN: error: <unknown>:0:0: in function invalid_load void (): Unsupported atomic address space
-
-name: invalid_load
-tracksRegLiveness: false
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr0 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr2_sgpr3
- $vgpr1 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $sgpr2_sgpr3, implicit $exec
- renamable $vgpr2 = FLAT_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load syncscope("one-as") seq_cst (s32) from `ptr addrspace(42) poison`)
- $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr poison`)
- S_ENDPGM 0
-
-...
----
-
-# GCN: error: <unknown>:0:0: in function invalid_store void (): Unsupported atomic address space
-
-name: invalid_store
-tracksRegLiveness: false
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (volatile store syncscope("agent-one-as") seq_cst (s32) into `ptr addrspace(42) poison`)
- S_ENDPGM 0
-
-...
----
-
-# GCN: error: <unknown>:0:0: in function invalid_cmpxchg void (): Unsupported atomic address space
-
-name: invalid_cmpxchg
-tracksRegLiveness: false
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr3 = V_MOV_B32_e32 $sgpr1, implicit $exec, implicit-def $vgpr2_vgpr3, implicit $sgpr0_sgpr1
- $vgpr0 = V_MOV_B32_e32 killed $sgpr4, implicit $exec, implicit $exec
- $vgpr1 = V_MOV_B32_e32 killed $sgpr5, implicit $exec, implicit $exec
- $vgpr2 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
- FLAT_ATOMIC_CMPSWAP killed renamable $vgpr2_vgpr3, killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load store syncscope("workgroup-one-as") seq_cst seq_cst (s32) on `ptr addrspace(42) poison`)
- S_ENDPGM 0
-
-...
----
-
-# GCN: error: <unknown>:0:0: in function invalid_rmw void (): Unsupported atomic address space
-
-name: invalid_rmw
-tracksRegLiveness: false
-body: |
- bb.0:
- liveins: $sgpr0_sgpr1
-
- $vgpr0 = V_MOV_B32_e32 $sgpr2, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr2_sgpr3
- $vgpr1 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $sgpr2_sgpr3, implicit $exec
- $vgpr2 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
- FLAT_ATOMIC_SWAP killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load store syncscope("wavefront-one-as") seq_cst (s32) on `ptr addrspace(42) poison`)
- S_ENDPGM 0
-
-...
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 20af74faa5c193..f8567bc3b0c5af 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -96,8 +96,11 @@ define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_ubyte v1, v0, s[2:3] sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_load_ubyte v1, v0, s[4:5] offset:8 sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_load_ubyte v1, v0, s[0:1] sc0 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: ; kill: killed $sgpr0_sgpr1
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
>From 72f56d6a41f6119b57c18458e8ed0890403cf69b Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Tue, 20 Oct 2026 03:44:04 -0400
Subject: [PATCH 3/5] Pass Function and DebugLoc instead of a Diag function to
getSynchronizeAddrSpace
Implemented by AI, manually checked.
Assisted-by: AI
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 13 +++++++------
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 8 +++++---
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 10 ++--------
3 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index d1583575afe422..ac90184508561e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -14,6 +14,7 @@
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -510,8 +511,8 @@ static const StringMap<SIAtomicAddrSpace> ASNames = {{
}};
std::optional<SIAtomicAddrSpace>
-getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
- function_ref<void(StringRef)> Diag) {
+getSynchronizeAddrSpace(const MMRAMetadata &MMRA, const Function *DiagFn,
+ const DebugLoc &DL) {
static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
if (!MMRA)
@@ -524,14 +525,15 @@ getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
if (auto It = ASNames.find(Suffix); It != ASNames.end()) {
Result |= It->second;
- } else {
+ } else if (DiagFn) {
SmallString<128> Str;
raw_svector_ostream OS(Str);
OS << "unknown address space '" << Suffix << "'; expected one of ";
ListSeparator LS;
for (const auto &[Name, Val] : ASNames)
OS << LS << '\'' << Name << '\'';
- Diag(Str);
+ DiagFn->getContext().diagnose(
+ DiagnosticInfoUnsupported(*DiagFn, Str, DL, DS_Warning));
}
}
@@ -542,8 +544,7 @@ getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
}
std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I) {
- const auto MMRA = MMRAMetadata(I);
- return getSynchronizeAddrSpace(MMRA, [](StringRef) {});
+ return getSynchronizeAddrSpace(MMRAMetadata(I));
}
} // end namespace llvm::AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index e4bc43de91da80..34860757d04c64 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -15,6 +15,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/IR/DebugLoc.h"
#include <optional>
namespace llvm {
@@ -152,11 +153,12 @@ std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I);
/// Parses the values for the "amdgpu-synchronize-as" MMRA.
/// If this tag isn't present, or if it has no meaningful values, returns
/// \p none, otherwise returns the address spaces specified by the MD.
-/// If an unexpected suffix for the MMRA tag is found, \p Diag is called with
-/// an error message.
+/// If \p DiagFn is non-null, unexpected suffixes for the MMRA tag are reported
+/// as warnings to \p DiagFn's context, located at \p DL.
std::optional<SIAtomicAddrSpace>
getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
- function_ref<void(StringRef)> Diag);
+ const Function *DiagFn = nullptr,
+ const DebugLoc &DL = {});
} // end namespace AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index dd84db28c66a06..3d65ce3f0e28ed 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -678,14 +678,8 @@ class SIMemoryLegalizerLegacy final : public MachineFunctionPass {
/// Issues a diagnostic if an unexpected suffix is observed for the MMRA tag.
static std::optional<SIAtomicAddrSpace>
getSynchronizeAddrSpaceMD(const MachineInstr &MI) {
- const auto MMRA = MMRAMetadata(MI.getMMRAMetadata());
- auto Diag = [&MI](StringRef S) {
- const MachineFunction *MF = MI.getMF();
- const Function &Fn = MF->getFunction();
- Fn.getContext().diagnose(
- DiagnosticInfoUnsupported(Fn, S, MI.getDebugLoc(), DS_Warning));
- };
- return getSynchronizeAddrSpace(MMRA, Diag);
+ return getSynchronizeAddrSpace(MMRAMetadata(MI.getMMRAMetadata()),
+ &MI.getMF()->getFunction(), MI.getDebugLoc());
}
static void diagnoseUnknownAVMetadata(const MachineInstr &MI,
>From 616c0a727d999bd7bdc8a7ed346efbe87e75a1ff Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Tue, 27 Oct 2026 06:37:10 -0400
Subject: [PATCH 4/5] Remove std::optional from getSynchronizeAddrSpace's
return type
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 14 +++++++-------
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 13 ++++++-------
llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 9 +++------
3 files changed, 16 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index ac90184508561e..f29af5f1d880a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -510,15 +510,15 @@ static const StringMap<SIAtomicAddrSpace> ASNames = {{
{"local", SIAtomicAddrSpace::LDS},
}};
-std::optional<SIAtomicAddrSpace>
-getSynchronizeAddrSpace(const MMRAMetadata &MMRA, const Function *DiagFn,
- const DebugLoc &DL) {
+SIAtomicAddrSpace getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
+ const Function *DiagFn,
+ const DebugLoc &DL) {
static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
if (!MMRA)
- return std::nullopt;
+ return SIAtomicAddrSpace::ATOMIC;
- SIAtomicAddrSpace Result = SIAtomicAddrSpace::NONE;
+ SIAtomicAddrSpace Result{};
for (const auto &[Prefix, Suffix] : MMRA) {
if (Prefix != FenceASPrefix)
continue;
@@ -538,12 +538,12 @@ getSynchronizeAddrSpace(const MMRAMetadata &MMRA, const Function *DiagFn,
}
if (Result == SIAtomicAddrSpace::NONE)
- return std::nullopt;
+ return SIAtomicAddrSpace::ATOMIC;
return Result;
}
-std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I) {
+SIAtomicAddrSpace getSynchronizeAddrSpace(const Instruction &I) {
return getSynchronizeAddrSpace(MMRAMetadata(I));
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index 34860757d04c64..79371b8b7767d5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -146,19 +146,18 @@ SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned AS);
/// Reads \p I's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
/// If this tag isn't present, or if it has no meaningful values, returns
-/// \p none, otherwise returns the address spaces specified by the MD.
+/// \p ATOMIC, otherwise returns the address spaces specified by the MD.
/// Does not diagnose unexpected MMRA tag suffixes.
-std::optional<SIAtomicAddrSpace> getSynchronizeAddrSpace(const Instruction &I);
+SIAtomicAddrSpace getSynchronizeAddrSpace(const Instruction &I);
/// Parses the values for the "amdgpu-synchronize-as" MMRA.
/// If this tag isn't present, or if it has no meaningful values, returns
-/// \p none, otherwise returns the address spaces specified by the MD.
+/// \p ATOMIC, otherwise returns the address spaces specified by the MD.
/// If \p DiagFn is non-null, unexpected suffixes for the MMRA tag are reported
/// as warnings to \p DiagFn's context, located at \p DL.
-std::optional<SIAtomicAddrSpace>
-getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
- const Function *DiagFn = nullptr,
- const DebugLoc &DL = {});
+SIAtomicAddrSpace getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
+ const Function *DiagFn = nullptr,
+ const DebugLoc &DL = {});
} // end namespace AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 3d65ce3f0e28ed..c2bffca49e0909 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -674,10 +674,9 @@ class SIMemoryLegalizerLegacy final : public MachineFunctionPass {
/// Reads \p MI's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
/// If this tag isn't present, or if it has no meaningful values, returns
-/// \p none, otherwise returns the address spaces specified by the MD.
+/// \p ATOMIC, otherwise returns the address spaces specified by the MD.
/// Issues a diagnostic if an unexpected suffix is observed for the MMRA tag.
-static std::optional<SIAtomicAddrSpace>
-getSynchronizeAddrSpaceMD(const MachineInstr &MI) {
+static SIAtomicAddrSpace getSynchronizeAddrSpaceMD(const MachineInstr &MI) {
return getSynchronizeAddrSpace(MMRAMetadata(MI.getMMRAMetadata()),
&MI.getMF()->getFunction(), MI.getDebugLoc());
}
@@ -902,9 +901,7 @@ SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const {
return std::nullopt;
}
- auto SynchronizeAS = getSynchronizeAddrSpaceMD(*MI);
- if (SynchronizeAS)
- OrderingAddrSpace = *SynchronizeAS;
+ OrderingAddrSpace = getSynchronizeAddrSpaceMD(*MI);
return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace,
SIAtomicAddrSpace::ATOMIC, IsCrossAddressSpaceOrdering,
>From 8c23c96248797c74f7de7563b85b58222920effa Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 28 Oct 2026 09:12:43 -0400
Subject: [PATCH 5/5] [NFC] Remove some obsolete includes
They were needed for intermediate states of this PR, but the current
final state doesn't need them any more.
---
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
index 79371b8b7767d5..e8a528ff69f59e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h
@@ -14,9 +14,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/IR/DebugLoc.h"
-#include <optional>
namespace llvm {
More information about the llvm-branch-commits
mailing list