[llvm-branch-commits] [llvm] [CodeGen][AMDGPU] Allow elementwise atomic load/store at element alignment (PR #219906)
Harrison Hao via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Aug 31 00:49:57 PDT 2026
https://github.com/harrisonGPU created https://github.com/llvm/llvm-project/pull/219906
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 13787e6cffa73ac8098eee2d6eefa605b546fd5f Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Mon, 31 Aug 2026 12:55:56 +0800
Subject: [PATCH] [CodeGen][AMDGPU] Allow elementwise atomic load/store at
element alignment
---
llvm/include/llvm/CodeGen/TargetLowering.h | 13 +-
llvm/lib/CodeGen/AtomicExpandPass.cpp | 41 +++++-
.../SelectionDAG/SelectionDAGBuilder.cpp | 10 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 17 +++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 40 ++++++
.../test/CodeGen/AMDGPU/load-atomic-global.ll | 120 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll | 24 ++++
.../CodeGen/AMDGPU/store-atomic-global.ll | 72 +++++++++++
.../AtomicExpand/AMDGPU/unaligned-atomic.ll | 37 ++++++
10 files changed, 366 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index d8d371dfe9910..d491337f71ea7 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2280,11 +2280,14 @@ class LLVM_ABI TargetLoweringBase {
unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; }
/// Return true if the target supports an atomic access of \p SizeInBytes
- /// bytes at the given \p Alignment. The default implementation only allows
- /// naturally aligned atomics, unless setSupportsUnalignedAtomics(true) was
- /// called.
- virtual bool supportsAtomicAlignment(Align Alignment,
- uint64_t SizeInBytes) const {
+ /// bytes at the given \p Alignment in \p AddrSpace. \p ElementSizeInBytes is
+ /// the granularity at which the access has to be indivisible; it is smaller
+ /// than \p SizeInBytes only for elementwise atomics. The default
+ /// implementation ignores both and only allows naturally aligned atomics,
+ /// unless setSupportsUnalignedAtomics(true) was called.
+ virtual bool supportsAtomicAlignment(Align Alignment, uint64_t SizeInBytes,
+ uint64_t ElementSizeInBytes,
+ unsigned AddrSpace) const {
return SupportsUnalignedAtomics || Alignment.value() >= SizeInBytes;
}
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index f2ffa40030cc0..93a3949af454e 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -230,6 +230,37 @@ static unsigned getAtomicOpSize(AtomicCmpXchgInst *CASI) {
return DL.getTypeStoreSize(CASI->getCompareOperand()->getType());
}
+static unsigned getAtomicOpElementSize(LoadInst *LI) {
+ if (!LI->isElementwise())
+ return getAtomicOpSize(LI);
+ return LI->getDataLayout().getTypeStoreSize(LI->getType()->getScalarType());
+}
+
+static unsigned getAtomicOpElementSize(StoreInst *SI) {
+ if (!SI->isElementwise())
+ return getAtomicOpSize(SI);
+ return SI->getDataLayout().getTypeStoreSize(
+ SI->getValueOperand()->getType()->getScalarType());
+}
+
+template <typename Inst> static unsigned getAtomicOpElementSize(Inst *I) {
+ return getAtomicOpSize(I);
+}
+
+/// Return the size \p I has to be aligned to. The element size is smaller than
+/// the whole access only for elementwise atomics, and only counts if the target
+/// can issue them at element alignment.
+template <typename Inst>
+static unsigned getRequiredAtomicSize(const TargetLowering *TLI, Inst *I) {
+ unsigned Size = getAtomicOpSize(I);
+ unsigned ElementSize = getAtomicOpElementSize(I);
+ if (ElementSize != Size &&
+ TLI->supportsAtomicAlignment(I->getAlign(), Size, ElementSize,
+ I->getPointerAddressSpace()))
+ return ElementSize;
+ return Size;
+}
+
/// Copy metadata that's safe to preserve when widening atomics.
static void copyMetadataForAtomic(Instruction &Dest,
const Instruction &Source) {
@@ -265,22 +296,22 @@ static void copyMetadataForAtomic(Instruction &Dest,
template <typename Inst>
static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
- unsigned Size = getAtomicOpSize(I);
- Align Alignment = I->getAlign();
unsigned MaxSize = TLI->getMaxAtomicSizeInBitsSupported() / 8;
- return Alignment >= Size && Size <= MaxSize;
+ return I->getAlign() >= getRequiredAtomicSize(TLI, I) &&
+ getAtomicOpSize(I) <= MaxSize;
}
template <typename Inst>
static void writeUnsupportedAtomicSizeReason(const TargetLowering *TLI, Inst *I,
raw_ostream &OS) {
+ unsigned RequiredSize = getRequiredAtomicSize(TLI, I);
unsigned Size = getAtomicOpSize(I);
Align Alignment = I->getAlign();
bool NeedSeparator = false;
- if (Alignment < Size) {
+ if (Alignment < RequiredSize) {
OS << "instruction alignment " << Alignment.value()
- << " is smaller than the required " << Size
+ << " is smaller than the required " << RequiredSize
<< "-byte alignment for this atomic operation";
NeedSeparator = true;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 367fa58ab94a1..5805c1bcdcd5a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5368,8 +5368,11 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
+ EVT ElementVT = I.isElementwise() ? MemVT.getScalarType() : MemVT;
- if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8))
+ if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8,
+ ElementVT.getSizeInBits() / 8,
+ I.getPointerAddressSpace()))
report_fatal_error("Cannot generate unaligned atomic load");
auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
@@ -5404,8 +5407,11 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT MemVT =
TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
+ EVT ElementVT = I.isElementwise() ? MemVT.getScalarType() : MemVT;
- if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8))
+ if (!TLI.supportsAtomicAlignment(I.getAlign(), MemVT.getSizeInBits() / 8,
+ ElementVT.getSizeInBits() / 8,
+ I.getPointerAddressSpace()))
report_fatal_error("Cannot generate unaligned atomic store");
auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d9810e3d9fbd9..5e9244cad7cff 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2361,6 +2361,23 @@ bool SITargetLowering::allowsMisalignedMemoryAccesses(
Alignment, Flags, IsFast);
}
+bool SITargetLowering::supportsAtomicAlignment(Align Alignment,
+ uint64_t SizeInBytes,
+ uint64_t ElementSizeInBytes,
+ unsigned AddrSpace) const {
+ // Each naturally aligned dword is separately atomic. Only global and flat
+ // opt in: the LDS 4 byte rule in allowsMisalignedMemoryAccessesImpl assumes
+ // lowering to ds_read2_b32, which never happens for an atomic, and buffer
+ // accesses need natural alignment for out-of-bounds semantics.
+ if (ElementSizeInBytes < SizeInBytes && ElementSizeInBytes >= 4 &&
+ (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace) ||
+ AddrSpace == AMDGPUAS::FLAT_ADDRESS))
+ return Alignment.value() >= ElementSizeInBytes;
+
+ return TargetLoweringBase::supportsAtomicAlignment(
+ Alignment, SizeInBytes, ElementSizeInBytes, AddrSpace);
+}
+
EVT SITargetLowering::getOptimalMemOpType(
LLVMContext &Context, const MemOp &Op,
const AttributeList &FuncAttributes) const {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 8ece9f279a100..d7e37825550e0 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -391,6 +391,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
unsigned *IsFast = nullptr) const override;
+ bool supportsAtomicAlignment(Align Alignment, uint64_t SizeInBytes,
+ uint64_t ElementSizeInBytes,
+ unsigned AddrSpace) const override;
+
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
const AttributeList &FuncAttributes) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
index 69e3378ae08b9..5e9a593de9a74 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -48,6 +48,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, pt
ret void
}
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_elementwise(ptr addrspace(0) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic elementwise <2 x float>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index 5ac3ca4786e92..4860f6b7cad1a 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -48,6 +48,86 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, pt
ret void
}
+define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_elementwise(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off glc dlc
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic elementwise <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_load_f32x2_unordered_agent_elementwise(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_unordered_agent_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_unordered_agent_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_unordered_agent_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_unordered_agent_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic elementwise <2 x float>, ptr addrspace(1) %p syncscope("agent") unordered, align 4
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
@@ -134,6 +214,46 @@ define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p
ret void
}
+define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront_elementwise(ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: global_store_dword v[2:3], v0, off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: global_store_dword v[2:3], v0, off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: global_store_b32 v[2:3], v0, off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX12-NEXT: global_store_b32 v[2:3], v0, off
+; GFX12-NEXT: s_endpgm
+ %a0 = load atomic elementwise <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
+ %num1 = extractelement <2 x float> %a0, i32 0
+ %num2 = extractelement <2 x float> %a0, i32 1
+ %res = fadd float %num1, %num2
+ store float %res, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
index 855091bb8a005..e2678e204d557 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll
@@ -32,6 +32,30 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
ret void
}
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent_elementwise(<2 x float> %in, ptr addrspace(0) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic elementwise <2 x float> %in, ptr addrspace(0) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(0) %out) {
; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
index 3de24b553f793..9ff69256f1b92 100644
--- a/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-atomic-global.ll
@@ -32,6 +32,54 @@ define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr a
ret void
}
+define amdgpu_cs void @atomic_store_f32x2_monotonic_agent_elementwise(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_agent_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
+; GFX12-NEXT: s_endpgm
+ store atomic elementwise <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
+ ret void
+}
+
+define amdgpu_cs void @atomic_store_f32x2_unordered_agent_elementwise(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_unordered_agent_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_unordered_agent_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_unordered_agent_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_unordered_agent_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
+ store atomic elementwise <2 x float> %in, ptr addrspace(1) %out syncscope("agent") unordered, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX9: ; %bb.0:
@@ -80,6 +128,30 @@ define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr
ret void
}
+define amdgpu_cs void @atomic_store_f32x2_monotonic_wavefront_elementwise(<2 x float> %in, ptr addrspace(1) %out) {
+; GFX9-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: s_endpgm
+;
+; GFX10-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX10-NEXT: s_endpgm
+;
+; GFX11-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
+;
+; GFX12-LABEL: atomic_store_f32x2_monotonic_wavefront_elementwise:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX12-NEXT: s_endpgm
+ store atomic elementwise <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
+ ret void
+}
+
define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX9: ; %bb.0:
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll
index e8884970c33e5..38a156684ff14 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/unaligned-atomic.ll
@@ -11,3 +11,40 @@ define void @atomic_store_global_align1(ptr addrspace(1) %ptr, i32 %val) {
store atomic i32 %val, ptr addrspace(1) %ptr monotonic, align 1
ret void
}
+
+; CHECK: error: unsupported atomic load: instruction alignment 2 is smaller than the required 8-byte alignment for this atomic operation
+define <2 x float> @atomic_load_global_elementwise_align2(ptr addrspace(1) %ptr) {
+ %val = load atomic elementwise <2 x float>, ptr addrspace(1) %ptr monotonic, align 2
+ ret <2 x float> %val
+}
+
+; CHECK: error: unsupported atomic store: instruction alignment 2 is smaller than the required 8-byte alignment for this atomic operation
+define void @atomic_store_global_elementwise_align2(ptr addrspace(1) %ptr, <2 x float> %val) {
+ store atomic elementwise <2 x float> %val, ptr addrspace(1) %ptr monotonic, align 2
+ ret void
+}
+
+; CHECK: error: unsupported atomic load: instruction alignment 4 is smaller than the required 8-byte alignment for this atomic operation
+define <2 x float> @atomic_load_global_align4(ptr addrspace(1) %ptr) {
+ %val = load atomic <2 x float>, ptr addrspace(1) %ptr monotonic, align 4
+ ret <2 x float> %val
+}
+
+; CHECK: error: unsupported atomic load: instruction alignment 2 is smaller than the required 4-byte alignment for this atomic operation
+define <2 x half> @atomic_load_global_elementwise_f16_align2(ptr addrspace(1) %ptr) {
+ %val = load atomic elementwise <2 x half>, ptr addrspace(1) %ptr monotonic, align 2
+ ret <2 x half> %val
+}
+
+; The LDS 4-byte rule assumes lowering to ds_read2_b32, which atomics never get.
+; CHECK: error: unsupported atomic load: instruction alignment 4 is smaller than the required 8-byte alignment for this atomic operation
+define <2 x float> @atomic_load_local_elementwise_align4(ptr addrspace(3) %ptr) {
+ %val = load atomic elementwise <2 x float>, ptr addrspace(3) %ptr monotonic, align 4
+ ret <2 x float> %val
+}
+
+; CHECK: error: unsupported atomic load: target supports atomics up to 8 bytes, but this atomic accesses 16 bytes
+define <4 x float> @atomic_load_global_elementwise_v4f32_align4(ptr addrspace(1) %ptr) {
+ %val = load atomic elementwise <4 x float>, ptr addrspace(1) %ptr monotonic, align 4
+ ret <4 x float> %val
+}
More information about the llvm-branch-commits
mailing list