[llvm] [SPIR-V] Honor explicit memory_order/memory_scope in atomic load/store (PR #215572)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 23:42:17 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/215572
>From 0b81f98c66ca34cd9c544487218df4b9904ab9cb Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 16:01:57 +0200
Subject: [PATCH 1/2] [SPIR-V] Honor explicit memory_order/memory_scope in
atomic load/store
atomic_load_explicit and atomic_store_explicit ignored their memory_order/memory_scope arguments
Translate the explicit arguments via the existing buildMemSemanticsReg/buildScopeReg helpers, as buildAtomicRMWInst already does
---
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 44 ++++++++++---------
.../test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll | 15 +++++--
2 files changed, 36 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 89b0292bc656b..38c770aed5836 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -635,24 +635,22 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
if (Call->isSpirvOp())
return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicLoad, Call, TypeReg);
+ // atomic_load_explicit(ptr, memory_order[, memory_scope]).
Register PtrRegister = Call->Arguments[0];
- // TODO: if true insert call to __translate_ocl_memory_sccope before
- // OpAtomicLoad and the function implementation. We can use Translator's
- // output for transcoding/atomic_explicit_arguments.cl as an example.
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+
+ unsigned Semantics =
+ SPIRV::MemorySemantics::SequentiallyConsistent |
+ getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
+ Register MemSemanticsReg =
+ Call->Arguments.size() >= 2 ? Call->Arguments[1] : Register();
+ MemSemanticsReg = buildMemSemanticsReg(MemSemanticsReg, PtrRegister,
+ Semantics, MIRBuilder, GR);
+
Register ScopeRegister =
- Call->Arguments.size() > 1
- ? Call->Arguments[1]
- : buildConstantIntReg32(SPIRV::Scope::Device, MIRBuilder, GR);
- Register MemSemanticsReg;
- if (Call->Arguments.size() > 2) {
- // TODO: Insert call to __translate_ocl_memory_order before OpAtomicLoad.
- MemSemanticsReg = Call->Arguments[2];
- } else {
- int Semantics =
- SPIRV::MemorySemantics::SequentiallyConsistent |
- getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
- MemSemanticsReg = buildConstantIntReg32(Semantics, MIRBuilder, GR);
- }
+ Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();
+ ScopeRegister =
+ buildScopeReg(ScopeRegister, SPIRV::Scope::Device, MIRBuilder, GR, MRI);
MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
.addDef(Call->ReturnRegister)
@@ -671,13 +669,19 @@ static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicStore, Call,
Register(0));
- Register ScopeRegister =
- buildConstantIntReg32(SPIRV::Scope::Device, MIRBuilder, GR);
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register PtrRegister = Call->Arguments[0];
- int Semantics =
+ unsigned Semantics =
SPIRV::MemorySemantics::SequentiallyConsistent |
getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
- Register MemSemanticsReg = buildConstantIntReg32(Semantics, MIRBuilder, GR);
+ Register MemSemanticsReg =
+ Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();
+ MemSemanticsReg = buildMemSemanticsReg(MemSemanticsReg, PtrRegister,
+ Semantics, MIRBuilder, GR);
+ Register ScopeRegister =
+ Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register();
+ ScopeRegister =
+ buildScopeReg(ScopeRegister, SPIRV::Scope::Device, MIRBuilder, GR, MRI);
MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
.addUse(PtrRegister)
.addUse(ScopeRegister)
diff --git a/llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll b/llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll
index cb81df5b61019..4c41fe3c92694 100644
--- a/llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll
+++ b/llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll
@@ -3,12 +3,21 @@
;; Types:
; CHECK: %[[#F32:]] = OpTypeFloat 32
+; CHECK: %[[#I32:]] = OpTypeInt 32 0
;; Constants:
-; CHECK: %[[#CONST:]] = OpConstant %[[#F32]] 1
+; CHECK-DAG: %[[#CONST:]] = OpConstant %[[#F32]] 1
+; CHECK-DAG: %[[#RELAXED:]] = OpConstantNull %[[#I32]]
+; CHECK-DAG: %[[#DEVICE:]] = OpConstant %[[#I32]] 1
+; CHECK-DAG: %[[#WORKGROUP:]] = OpConstant %[[#I32]] 2
+; CHECK-DAG: %[[#SEQCST:]] = OpConstant %[[#I32]] 16
;; Atomic instructions:
; CHECK: OpStore %[[#]] %[[#CONST]]
-; CHECK-COUNT-3: OpAtomicStore
-; CHECK-COUNT-3: OpAtomicLoad
+; CHECK: OpAtomicStore %[[#]] %[[#DEVICE]] %[[#SEQCST]] %[[#CONST]]
+; CHECK: OpAtomicStore %[[#]] %[[#DEVICE]] %[[#RELAXED]] %[[#CONST]]
+; CHECK: OpAtomicStore %[[#]] %[[#WORKGROUP]] %[[#RELAXED]] %[[#CONST]]
+; CHECK: OpAtomicLoad %[[#]] %[[#]] %[[#DEVICE]] %[[#SEQCST]]
+; CHECK: OpAtomicLoad %[[#]] %[[#]] %[[#DEVICE]] %[[#RELAXED]]
+; CHECK: OpAtomicLoad %[[#]] %[[#]] %[[#WORKGROUP]] %[[#RELAXED]]
; CHECK-COUNT-3: OpAtomicExchange
define spir_kernel void @test_atomic_kernel(ptr addrspace(3) %ff) local_unnamed_addr #0 !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
>From f2103dd5fb23aaf54bf206dd16962217fb33d1f1 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 13 Aug 2026 08:42:06 +0200
Subject: [PATCH 2/2] Address comment
---
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 61 ++++++++++++++-----------
1 file changed, 35 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 38c770aed5836..08dd26ac47b1c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -579,23 +579,31 @@ static void setRegClassIfNull(Register Reg, MachineRegisterInfo *MRI,
SpvType ? GR->getRegClass(SpvType) : &SPIRV::iIDRegClass);
}
+/// Storage-class bits are only merged in for an explicit memory_order arg;
+/// DefaultSemantics is returned as-is otherwise.
+static unsigned computeMemSemantics(Register SemanticsRegister,
+ Register PtrRegister,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR,
+ unsigned DefaultSemantics) {
+ if (!SemanticsRegister.isValid())
+ return DefaultSemantics;
+ std::memory_order Order = static_cast<std::memory_order>(
+ getIConstVal(SemanticsRegister, MIRBuilder.getMRI()));
+ return getSPIRVMemSemantics(Order) |
+ getMemSemanticsForStorageClass(
+ GR->getPointerStorageClass(PtrRegister));
+}
+
static Register buildMemSemanticsReg(Register SemanticsRegister,
- Register PtrRegister, unsigned &Semantics,
+ Register PtrRegister,
MachineIRBuilder &MIRBuilder,
- SPIRVGlobalRegistry *GR) {
- if (SemanticsRegister.isValid()) {
- MachineRegisterInfo *MRI = MIRBuilder.getMRI();
- std::memory_order Order =
- static_cast<std::memory_order>(getIConstVal(SemanticsRegister, MRI));
- Semantics =
- getSPIRVMemSemantics(Order) |
- getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
- if (static_cast<unsigned>(Order) == Semantics) {
- MRI->setRegClass(SemanticsRegister, &SPIRV::iIDRegClass);
- return SemanticsRegister;
- }
- }
- return buildConstantIntReg32(Semantics, MIRBuilder, GR);
+ SPIRVGlobalRegistry *GR,
+ unsigned DefaultSemantics) {
+ return buildConstantIntReg32(computeMemSemantics(SemanticsRegister,
+ PtrRegister, MIRBuilder, GR,
+ DefaultSemantics),
+ MIRBuilder, GR);
}
static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode,
@@ -639,13 +647,13 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
Register PtrRegister = Call->Arguments[0];
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
- unsigned Semantics =
+ unsigned DefaultSemantics =
SPIRV::MemorySemantics::SequentiallyConsistent |
getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
Register MemSemanticsReg =
Call->Arguments.size() >= 2 ? Call->Arguments[1] : Register();
MemSemanticsReg = buildMemSemanticsReg(MemSemanticsReg, PtrRegister,
- Semantics, MIRBuilder, GR);
+ MIRBuilder, GR, DefaultSemantics);
Register ScopeRegister =
Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();
@@ -671,13 +679,13 @@ static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register PtrRegister = Call->Arguments[0];
- unsigned Semantics =
+ unsigned DefaultSemantics =
SPIRV::MemorySemantics::SequentiallyConsistent |
getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
Register MemSemanticsReg =
Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();
MemSemanticsReg = buildMemSemanticsReg(MemSemanticsReg, PtrRegister,
- Semantics, MIRBuilder, GR);
+ MIRBuilder, GR, DefaultSemantics);
Register ScopeRegister =
Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register();
ScopeRegister =
@@ -809,11 +817,11 @@ static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
MIRBuilder, GR, MRI);
Register PtrRegister = Call->Arguments[0];
- unsigned Semantics = SPIRV::MemorySemantics::None;
Register MemSemanticsReg =
Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();
- MemSemanticsReg = buildMemSemanticsReg(MemSemanticsReg, PtrRegister,
- Semantics, MIRBuilder, GR);
+ MemSemanticsReg =
+ buildMemSemanticsReg(MemSemanticsReg, PtrRegister, MIRBuilder, GR,
+ SPIRV::MemorySemantics::None);
Register ValueReg = Call->Arguments[1];
Register ValueTypeReg = GR->getSPIRVTypeID(Call->ReturnType);
// support cl_ext_float_atomics
@@ -881,11 +889,12 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register PtrRegister = Call->Arguments[0];
- unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent;
- Register MemSemanticsReg =
+ Register MemSemanticsArg =
Call->Arguments.size() >= 2 ? Call->Arguments[1] : Register();
- MemSemanticsReg = buildMemSemanticsReg(MemSemanticsReg, PtrRegister,
- Semantics, MIRBuilder, GR);
+ unsigned Semantics =
+ computeMemSemantics(MemSemanticsArg, PtrRegister, MIRBuilder, GR,
+ SPIRV::MemorySemantics::SequentiallyConsistent);
+ Register MemSemanticsReg = buildConstantIntReg32(Semantics, MIRBuilder, GR);
assert((Opcode != SPIRV::OpAtomicFlagClear ||
(Semantics != SPIRV::MemorySemantics::Acquire &&
More information about the llvm-commits
mailing list