[llvm] [SPIRV] Lower load/store atomic to OpAtomicLoad/OpAtomicStore (PR #185696)
Juan Manuel Martinez Caamaño via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 01:59:21 PDT 2026
https://github.com/jmmartinez updated https://github.com/llvm/llvm-project/pull/185696
>From 70d2cd6f39a02872c83a68f78e7a6b657b7a6df5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Tue, 10 Mar 2026 17:45:57 +0100
Subject: [PATCH 1/7] [SPIRV] Lower load/store atomic to
OpAtomicLoad/OpAtomicStore
---
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 4 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 6 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 8 +-
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 103 ++++++++++++++++--
llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp | 18 +++
llvm/lib/Target/SPIRV/SPIRVTypeInst.h | 8 ++
.../atomic-load-store-unsupported.ll | 41 +++++++
.../CodeGen/SPIRV/transcoding/load-atomic.ll | 44 ++++----
.../CodeGen/SPIRV/transcoding/store-atomic.ll | 45 ++++----
9 files changed, 211 insertions(+), 66 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/transcoding/atomic-load-store-unsupported.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index d2a5fa1f08724..a942f8ed464e2 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -24,8 +24,8 @@ let TargetPrefix = "spv" in {
def int_spv_unref_global : Intrinsic<[], [llvm_any_ty]>;
def int_spv_gep : Intrinsic<[llvm_any_ty], [llvm_i1_ty, llvm_any_ty, llvm_vararg_ty], [ImmArg<ArgIndex<0>>]>;
- def int_spv_load : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
- def int_spv_store : Intrinsic<[], [llvm_any_ty, llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ def int_spv_load : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty, llvm_i8_ty], [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ def int_spv_store : Intrinsic<[], [llvm_any_ty, llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty, llvm_i8_ty], [ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
def int_spv_extractv : Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_vararg_ty]>;
def int_spv_insertv : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_any_ty, llvm_vararg_ty]>;
def int_spv_extractelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_anyint_ty]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index a96210b955f23..4073c4ad41bf3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2239,7 +2239,8 @@ Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
{I.getPointerOperand(), B.getInt16(Flags),
- B.getInt32(I.getAlign().value())});
+ B.getInt32(I.getAlign().value()),
+ B.getInt8(static_cast<uint8_t>(I.getOrdering()))});
replaceMemInstrUses(&I, NewI, B);
return NewI;
}
@@ -2270,7 +2271,8 @@ Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
auto *NewI = B.CreateIntrinsic(
Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
{I.getValueOperand(), PtrOp, B.getInt16(Flags),
- B.getInt32(I.getAlign().value())});
+ B.getInt32(I.getAlign().value()),
+ B.getInt8(static_cast<uint8_t>(I.getOrdering()))});
NewI->copyMetadata(I);
I.eraseFromParent();
return NewI;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 1faf91fc52963..f6812fd775b6f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -115,15 +115,15 @@ void SPIRVTargetLowering::getTgtMemIntrinsic(
AlignIdx = 2;
[[fallthrough]];
case Intrinsic::spv_store: {
- if (I.getNumOperands() >= AlignIdx + 1) {
- auto *AlignOp = cast<ConstantInt>(I.getOperand(AlignIdx));
- Info.align = Align(AlignOp->getZExtValue());
- }
+ auto *AlignOp = cast<ConstantInt>(I.getOperand(AlignIdx));
+ Info.align = Align(AlignOp->getZExtValue());
Info.flags = static_cast<MachineMemOperand::Flags>(
cast<ConstantInt>(I.getOperand(AlignIdx - 1))->getZExtValue());
Info.memVT = MVT::i64;
// TODO: take into account opaque pointers (don't use getElementType).
// MVT::getVT(PtrTy->getElementType());
+ Info.order = static_cast<AtomicOrdering>(
+ cast<ConstantInt>(I.getOperand(AlignIdx + 1))->getZExtValue());
Infos.push_back(Info);
return;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 56aacaf63928c..ccf3e3e944a8b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -160,7 +160,13 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
+ bool selectAtomicLoad(Register ResVReg, SPIRVTypeInst ResType, Register Ptr,
+ const MachineMemOperand &MemOp,
+ MachineIRBuilder &MIRBuilder, MachineInstr &L) const;
bool selectStore(MachineInstr &I) const;
+ bool selectAtomicStore(Register StoreVal, Register Ptr,
+ const MachineMemOperand &MemOp,
+ MachineIRBuilder &MIRBuilder, MachineInstr &S) const;
bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
@@ -1900,7 +1906,15 @@ bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
}
}
- auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
+ MachineIRBuilder MIRBuilder(I);
+
+ if (I.getNumMemOperands()) {
+ const MachineMemOperand *MemOp = *I.memoperands_begin();
+ if (MemOp->isAtomic())
+ return selectAtomicLoad(ResVReg, ResType, Ptr, *MemOp, MIRBuilder, I);
+ }
+
+ auto MIB = MIRBuilder.buildInstr(SPIRV::OpLoad)
.addDef(ResVReg)
.addUse(GR.getSPIRVTypeID(ResType))
.addUse(Ptr);
@@ -1910,13 +1924,48 @@ bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
} else {
- MachineIRBuilder MIRBuilder(I);
addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
}
MIB.constrainAllUses(TII, TRI, RBI);
return true;
}
+bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg,
+ SPIRVTypeInst ResType,
+ Register Ptr,
+ const MachineMemOperand &MemOp,
+ MachineIRBuilder &MIRBuilder,
+ MachineInstr &L) const {
+ LLVMContext &Context = L.getMF()->getFunction().getContext();
+
+ if (!ResType.isTypeIntOrFloat())
+ return diagnoseUnsupported(
+ L, "atomic load is only allowed for integer or floating point types");
+
+ // This could be relaxed since the volatile attribute on atomic load is
+ // supported with the VulkanMemoryModelKHR capability.
+ if (MemOp.isVolatile())
+ return diagnoseUnsupported(
+ L, "atomic load of volatile memory is not supported");
+
+ uint32_t Scope =
+ static_cast<uint32_t>(getMemScope(Context, MemOp.getSyncScopeID()));
+ Register ScopeReg = buildI32Constant(Scope, L);
+
+ AtomicOrdering AO = MemOp.getSuccessOrdering();
+ uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
+ Register MemSemReg = buildI32Constant(MemSem, L);
+
+ auto AtomicLoad = MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
+ .addDef(ResVReg)
+ .addUse(GR.getSPIRVTypeID(ResType))
+ .addUse(Ptr)
+ .addUse(ScopeReg)
+ .addUse(MemSemReg);
+ AtomicLoad.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
Register StoreVal = I.getOperand(0 + OpOffset).getReg();
@@ -1952,23 +2001,63 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
}
}
- MachineBasicBlock &BB = *I.getParent();
- auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
- .addUse(Ptr)
- .addUse(StoreVal);
+ MachineIRBuilder MIRBuilder(I);
+
+ if (I.getNumMemOperands()) {
+ const MachineMemOperand *MemOp = *I.memoperands_begin();
+ if (MemOp->isAtomic())
+ return selectAtomicStore(StoreVal, Ptr, *MemOp, MIRBuilder, I);
+ }
+
+ auto MIB = MIRBuilder.buildInstr(SPIRV::OpStore).addUse(Ptr).addUse(StoreVal);
if (!I.getNumMemOperands()) {
assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
I.getOpcode() ==
TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
} else {
- MachineIRBuilder MIRBuilder(I);
addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
}
MIB.constrainAllUses(TII, TRI, RBI);
return true;
}
+bool SPIRVInstructionSelector::selectAtomicStore(Register StoreVal,
+ Register Ptr,
+ const MachineMemOperand &MemOp,
+ MachineIRBuilder &MIRBuilder,
+ MachineInstr &S) const {
+ LLVMContext &Context = S.getMF()->getFunction().getContext();
+
+ SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(Ptr);
+ SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
+ if (!PointeeType.isTypeIntOrFloat())
+ return diagnoseUnsupported(
+ S, "atomic store is only allowed for integer or floating point types");
+
+ // This could be relaxed since the volatile attribute on atomic store is
+ // supported with the VulkanMemoryModelKHR capability.
+ if (MemOp.isVolatile())
+ return diagnoseUnsupported(
+ S, "atomic store of volatile memory is not supported");
+
+ uint32_t Scope =
+ static_cast<uint32_t>(getMemScope(Context, MemOp.getSyncScopeID()));
+ Register ScopeReg = buildI32Constant(Scope, S);
+
+ AtomicOrdering AO = MemOp.getSuccessOrdering();
+ uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
+ Register MemSemReg = buildI32Constant(MemSem, S);
+
+ auto AtomicStore = MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
+ .addUse(Ptr)
+ .addUse(ScopeReg)
+ .addUse(MemSemReg)
+ .addUse(StoreVal);
+ AtomicStore.constrainAllUses(TII, TRI, RBI);
+ return true;
+}
+
bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg,
SPIRVTypeInst ResType,
MachineInstr &I) const {
diff --git a/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp b/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp
index 889dc749c5779..f78d4c9880387 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp
@@ -14,6 +14,8 @@
#include "MCTargetDesc/SPIRVMCTargetDesc.h"
#include "SPIRVInstrInfo.h"
+#include "SPIRV.h"
+
namespace llvm {
[[maybe_unused]] static bool definesATypeRegister(const MachineInstr &MI) {
const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
@@ -32,4 +34,20 @@ bool SPIRVTypeInst::isTypeIntN(unsigned N) const {
return MI->getOperand(1).getImm() == N;
return true;
}
+
+bool SPIRVTypeInst::isTypeFloat(unsigned Width,
+ std::optional<unsigned> Encoding) const {
+ if (MI->getOpcode() != SPIRV::OpTypeFloat)
+ return false;
+ if (Width)
+ return MI->getOperand(1).getImm() == Width;
+ if (Encoding) {
+ if (MI->getNumOperands() < 3)
+ return false;
+ const MachineOperand &EO = MI->getOperand(2);
+ assert(EO.isImm() && "Encoding operand must be an immediate");
+ return EO.getImm() == *Encoding;
+ }
+ return true;
+}
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVTypeInst.h b/llvm/lib/Target/SPIRV/SPIRVTypeInst.h
index 5b2dbac9b10c7..3fb93676f149c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTypeInst.h
+++ b/llvm/lib/Target/SPIRV/SPIRVTypeInst.h
@@ -51,6 +51,14 @@ class SPIRVTypeInst {
// Returns true if this is an OpTypeInt instruction.
// If N is non-zero, also checks that the bit width matches N.
bool isTypeIntN(unsigned N = 0) const;
+ // Returns true if this is an OpTypeFloat instruction.
+ // If Width is non-zero, also checks that the bit width matches Width.
+ // If Encoding is set, also checks that the optional
+ // encoding.
+ bool isTypeFloat(unsigned Width = 0,
+ std::optional<unsigned> Encoding = std::nullopt) const;
+ // Returns true if this is an OpTypeInt or OpTypeFloat instruction.
+ bool isTypeIntOrFloat() const { return isTypeIntN() || isTypeFloat(); }
friend struct DenseMapInfo<SPIRVTypeInst>;
};
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/atomic-load-store-unsupported.ll b/llvm/test/CodeGen/SPIRV/transcoding/atomic-load-store-unsupported.ll
new file mode 100644
index 0000000000000..4382c3cf87097
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/transcoding/atomic-load-store-unsupported.ll
@@ -0,0 +1,41 @@
+; Verify that atomic load/store of vectors and volatile atomic load/store
+; correctly fail to select.
+
+; RUN: split-file %s %t
+
+; RUN: not llc -O0 -mtriple=spirv64-- %t/load-vector.ll -o /dev/null 2>&1 | FileCheck --check-prefix=FAIL-LOAD-VEC %s
+
+; RUN: not llc -O0 -mtriple=spirv64-- %t/load-volatile.ll -o /dev/null 2>&1 | FileCheck --check-prefix=FAIL-LOAD-VOL %s
+
+; RUN: not llc -O0 -mtriple=spirv64-- %t/store-vector.ll -o /dev/null 2>&1 | FileCheck --check-prefix=FAIL-STORE-VEC %s
+
+; RUN: not llc -O0 -mtriple=spirv64-- %t/store-volatile.ll -o /dev/null 2>&1 | FileCheck --check-prefix=FAIL-STORE-VOL %s
+
+; FAIL-LOAD-VEC: error:{{.*}}atomic load is only allowed for integer or floating point types
+; FAIL-LOAD-VOL: error:{{.*}}atomic load of volatile memory is not supported
+; FAIL-STORE-VEC: error:{{.*}}atomic store is only allowed for integer or floating point types
+; FAIL-STORE-VOL: error:{{.*}}atomic store of volatile memory is not supported
+
+;--- load-vector.ll
+define <2 x i32> @load_vector_acquire(ptr addrspace(1) %ptr) {
+ %val = load atomic <2 x i32>, ptr addrspace(1) %ptr acquire, align 8
+ ret <2 x i32> %val
+}
+
+;--- load-volatile.ll
+define i32 @load_i32_acquire_device_volatile(ptr addrspace(1) %ptr) {
+ %val = load atomic volatile i32, ptr addrspace(1) %ptr syncscope("device") acquire, align 4
+ ret i32 %val
+}
+
+;--- store-vector.ll
+define void @store_vector_release(ptr addrspace(1) %ptr, <2 x i32> %val) {
+ store atomic <2 x i32> %val, ptr addrspace(1) %ptr release, align 8
+ ret void
+}
+
+;--- store-volatile.ll
+define void @store_i32_release_device_volatile(ptr addrspace(1) %ptr, i32 %val) {
+ store atomic volatile i32 %val, ptr addrspace(1) %ptr syncscope("device") release, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/load-atomic.ll b/llvm/test/CodeGen/SPIRV/transcoding/load-atomic.ll
index 0ebd3a5ec20ae..2a871c78bf0fb 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/load-atomic.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/load-atomic.ll
@@ -4,19 +4,24 @@
; RUN: llc -O0 -mtriple=spirv32-- %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-- %s -o - -filetype=obj | spirv-val %}
-;; Check that 'load atomic' LLVM IR instructions are lowered.
-;; NOTE: The current lowering is incorrect: 'load atomic' should produce
-;; OpAtomicLoad but currently produces OpLoad, silently dropping the atomic
-;; ordering. This test documents the broken behaviour so it can be fixed.
+; Check that 'load atomic' LLVM IR instructions are lowered correctly to
+; OpAtomicLoad with the right Scope and Memory Semantics operands.
+;
+; unordered and monotonic are currently mapped to Memory Semantics `None (Relaxed)` 0x0
; CHECK-DAG: %[[#Int32:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Float:]] = OpTypeFloat 32
-; CHECK-DAG: %[[#Int32Vec:]] = OpTypeVector %[[#Int32]] 2
+; CHECK-DAG: %[[#Const0:]] = OpConstantNull %[[#Int32]]
+; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#Int32]] 1{{$}}
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32]] 2{{$}}
+; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Int32]] 3{{$}}
+; CHECK-DAG: %[[#Const4:]] = OpConstant %[[#Int32]] 4{{$}}
+; CHECK-DAG: %[[#Const16:]] = OpConstant %[[#Int32]] 16{{$}}
define i32 @load_i32_unordered(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const0]] %[[#Const0]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr unordered, align 4
ret i32 %val
@@ -25,7 +30,7 @@ define i32 @load_i32_unordered(ptr addrspace(1) %ptr) {
define i32 @load_i32_monotonic(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const0]] %[[#Const0]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr monotonic, align 4
ret i32 %val
@@ -34,7 +39,7 @@ define i32 @load_i32_monotonic(ptr addrspace(1) %ptr) {
define i32 @load_i32_acquire(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const0]] %[[#Const2]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr acquire, align 4
ret i32 %val
@@ -43,7 +48,7 @@ define i32 @load_i32_acquire(ptr addrspace(1) %ptr) {
define i32 @load_i32_seq_cst(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const0]] %[[#Const16]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr seq_cst, align 4
ret i32 %val
@@ -54,7 +59,7 @@ define i32 @load_i32_seq_cst(ptr addrspace(1) %ptr) {
define i32 @load_i32_acquire_singlethread(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const4]] %[[#Const2]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr syncscope("singlethread") acquire, align 4
ret i32 %val
@@ -63,7 +68,7 @@ define i32 @load_i32_acquire_singlethread(ptr addrspace(1) %ptr) {
define i32 @load_i32_acquire_subgroup(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const3]] %[[#Const2]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr syncscope("subgroup") acquire, align 4
ret i32 %val
@@ -72,7 +77,7 @@ define i32 @load_i32_acquire_subgroup(ptr addrspace(1) %ptr) {
define i32 @load_i32_acquire_workgroup(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const2]] %[[#Const2]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr syncscope("workgroup") acquire, align 4
ret i32 %val
@@ -81,7 +86,7 @@ define i32 @load_i32_acquire_workgroup(ptr addrspace(1) %ptr) {
define i32 @load_i32_acquire_device(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 4
+; CHECK: %[[#]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const1]] %[[#Const2]]
; CHECK: OpReturnValue
%val = load atomic i32, ptr addrspace(1) %ptr syncscope("device") acquire, align 4
ret i32 %val
@@ -92,20 +97,9 @@ define i32 @load_i32_acquire_device(ptr addrspace(1) %ptr) {
define float @load_float_acquire(ptr addrspace(1) %ptr) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#load:]] = OpLoad %[[#Int32]] %[[#ptr]] Aligned 8
+; CHECK: %[[#load:]] = OpAtomicLoad %[[#Int32]] %[[#ptr]] %[[#Const0]] %[[#Const2]]
; CHECK: %[[#val:]] = OpBitcast %[[#Float]] %[[#load]]
; CHECK: OpReturnValue %[[#val]]
%val = load atomic float, ptr addrspace(1) %ptr acquire, align 8
ret float %val
}
-
-; -- test with a vector type
-
-define <2 x i32> @load_vector_acquire(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: OpFunction %[[#]]
-; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#]] = OpLoad %[[#Int32Vec]] %[[#ptr]] Aligned 8
-; CHECK: OpReturnValue
- %val = load atomic <2 x i32>, ptr addrspace(1) %ptr acquire, align 8
- ret <2 x i32> %val
-}
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/store-atomic.ll b/llvm/test/CodeGen/SPIRV/transcoding/store-atomic.ll
index b11b26451d086..7337a631a8616 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/store-atomic.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/store-atomic.ll
@@ -4,20 +4,25 @@
; RUN: llc -O0 -mtriple=spirv32-- %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-- %s -o - -filetype=obj | spirv-val %}
-;; Check that 'store atomic' LLVM IR instructions are lowered.
-;; NOTE: The current lowering is incorrect: 'store atomic' should produce
-;; OpAtomicStore but currently produces OpStore, silently dropping the atomic
-;; ordering. This test documents the broken behaviour so it can be fixed.
+; Check that 'store atomic' LLVM IR instructions are lowered correctly to
+; OpAtomicStore with the right Scope and Memory Semantics operands.
+;
+; unordered and monotonic are currently mapped to Memory Semantics `None (Relaxed)` 0x0
; CHECK-DAG: %[[#Int32:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Float:]] = OpTypeFloat 32
-; CHECK-DAG: %[[#Int32Vec:]] = OpTypeVector %[[#Int32]] 2
+; CHECK-DAG: %[[#Const0:]] = OpConstantNull %[[#Int32]]
+; CHECK-DAG: %[[#Const1:]] = OpConstant %[[#Int32]] 1{{$}}
+; CHECK-DAG: %[[#Const2:]] = OpConstant %[[#Int32]] 2{{$}}
+; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Int32]] 3{{$}}
+; CHECK-DAG: %[[#Const4:]] = OpConstant %[[#Int32]] 4{{$}}
+; CHECK-DAG: %[[#Const16:]] = OpConstant %[[#Int32]] 16{{$}}
define void @store_i32_unordered(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const0]] %[[#Const0]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr unordered, align 4
ret void
@@ -27,7 +32,7 @@ define void @store_i32_monotonic(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const0]] %[[#Const0]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr monotonic, align 4
ret void
@@ -37,7 +42,7 @@ define void @store_i32_release(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const0]] %[[#Const4]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr release, align 4
ret void
@@ -47,7 +52,7 @@ define void @store_i32_seq_cst(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const0]] %[[#Const16]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr seq_cst, align 4
ret void
@@ -59,7 +64,7 @@ define void @store_i32_release_singlethread(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const4]] %[[#Const4]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr syncscope("singlethread") release, align 4
ret void
@@ -69,7 +74,7 @@ define void @store_i32_release_subgroup(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const3]] %[[#Const4]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr syncscope("subgroup") release, align 4
ret void
@@ -79,7 +84,7 @@ define void @store_i32_release_workgroup(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const2]] %[[#Const4]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr syncscope("workgroup") release, align 4
ret void
@@ -89,7 +94,7 @@ define void @store_i32_release_device(ptr addrspace(1) %ptr, i32 %val) {
; CHECK-LABEL: OpFunction %[[#]]
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 4
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const1]] %[[#Const4]] %[[#val]]
; CHECK: OpReturn
store atomic i32 %val, ptr addrspace(1) %ptr syncscope("device") release, align 4
ret void
@@ -102,20 +107,8 @@ define void @store_float_release(ptr addrspace(1) %ptr, float %val) {
; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
; CHECK: %[[#val:]] = OpFunctionParameter %[[#Float]]
; CHECK: %[[#cast:]] = OpBitcast %[[#Int32]] %[[#val]]
-; CHECK: OpStore %[[#ptr]] %[[#cast]] Aligned 8
+; CHECK: OpAtomicStore %[[#ptr]] %[[#Const0]] %[[#Const4]] %[[#cast]]
; CHECK: OpReturn
store atomic float %val, ptr addrspace(1) %ptr release, align 8
ret void
}
-
-; -- test with a vector type
-
-define void @store_vector_release(ptr addrspace(1) %ptr, <2 x i32> %val) {
-; CHECK-LABEL: OpFunction %[[#]]
-; CHECK: %[[#ptr:]] = OpFunctionParameter %[[#]]
-; CHECK: %[[#val:]] = OpFunctionParameter %[[#Int32Vec]]
-; CHECK: OpStore %[[#ptr]] %[[#val]] Aligned 8
-; CHECK: OpReturn
- store atomic <2 x i32> %val, ptr addrspace(1) %ptr release, align 8
- ret void
-}
>From e0cccd6878c31556ea8bfe46fc0347f59302a16c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Mon, 20 Apr 2026 17:01:26 +0200
Subject: [PATCH 2/7] Review: simplify isTypeFloat function
---
llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp | 16 ++--------------
llvm/lib/Target/SPIRV/SPIRVTypeInst.h | 8 ++------
2 files changed, 4 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp b/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp
index f78d4c9880387..7a7d62f9297f5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTypeInst.cpp
@@ -35,19 +35,7 @@ bool SPIRVTypeInst::isTypeIntN(unsigned N) const {
return true;
}
-bool SPIRVTypeInst::isTypeFloat(unsigned Width,
- std::optional<unsigned> Encoding) const {
- if (MI->getOpcode() != SPIRV::OpTypeFloat)
- return false;
- if (Width)
- return MI->getOperand(1).getImm() == Width;
- if (Encoding) {
- if (MI->getNumOperands() < 3)
- return false;
- const MachineOperand &EO = MI->getOperand(2);
- assert(EO.isImm() && "Encoding operand must be an immediate");
- return EO.getImm() == *Encoding;
- }
- return true;
+bool SPIRVTypeInst::isAnyTypeFloat() const {
+ return MI->getOpcode() == SPIRV::OpTypeFloat;
}
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVTypeInst.h b/llvm/lib/Target/SPIRV/SPIRVTypeInst.h
index 3fb93676f149c..39648ef622d3e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTypeInst.h
+++ b/llvm/lib/Target/SPIRV/SPIRVTypeInst.h
@@ -52,13 +52,9 @@ class SPIRVTypeInst {
// If N is non-zero, also checks that the bit width matches N.
bool isTypeIntN(unsigned N = 0) const;
// Returns true if this is an OpTypeFloat instruction.
- // If Width is non-zero, also checks that the bit width matches Width.
- // If Encoding is set, also checks that the optional
- // encoding.
- bool isTypeFloat(unsigned Width = 0,
- std::optional<unsigned> Encoding = std::nullopt) const;
+ bool isAnyTypeFloat() const;
// Returns true if this is an OpTypeInt or OpTypeFloat instruction.
- bool isTypeIntOrFloat() const { return isTypeIntN() || isTypeFloat(); }
+ bool isTypeIntOrFloat() const { return isTypeIntN() || isAnyTypeFloat(); }
friend struct DenseMapInfo<SPIRVTypeInst>;
};
>From 28faab4e39ff8bb483cc2cb54a6f7c5d2df14109 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Tue, 21 Apr 2026 10:42:05 +0200
Subject: [PATCH 3/7] Review: use spv_atomic_load/store intrinsics
---
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 6 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 37 ++++++++---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 49 ++++++++++----
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 65 ++++++++++---------
4 files changed, 102 insertions(+), 55 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index a942f8ed464e2..07820880300d7 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -24,8 +24,10 @@ let TargetPrefix = "spv" in {
def int_spv_unref_global : Intrinsic<[], [llvm_any_ty]>;
def int_spv_gep : Intrinsic<[llvm_any_ty], [llvm_i1_ty, llvm_any_ty, llvm_vararg_ty], [ImmArg<ArgIndex<0>>]>;
- def int_spv_load : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty, llvm_i8_ty], [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
- def int_spv_store : Intrinsic<[], [llvm_any_ty, llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty, llvm_i8_ty], [ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
+ def int_spv_load : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
+ def int_spv_atomic_load : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i16_ty, llvm_i8_ty], [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
+ def int_spv_store : Intrinsic<[], [llvm_any_ty, llvm_anyptr_ty, llvm_i16_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ def int_spv_atomic_store : Intrinsic<[], [llvm_any_ty, llvm_anyptr_ty, llvm_i16_ty, llvm_i8_ty], [ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
def int_spv_extractv : Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_vararg_ty]>;
def int_spv_insertv : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_any_ty, llvm_vararg_ty]>;
def int_spv_extractelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_anyint_ty]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 4073c4ad41bf3..ef70f57d3b2e2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2236,11 +2236,19 @@ Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
MachineMemOperand::Flags Flags =
TLI->getLoadMemOperandFlags(I, CurrF->getDataLayout());
- auto *NewI =
- B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
- {I.getPointerOperand(), B.getInt16(Flags),
- B.getInt32(I.getAlign().value()),
- B.getInt8(static_cast<uint8_t>(I.getOrdering()))});
+
+ unsigned IntrinsicId;
+ SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(Flags)};
+ if (!I.isAtomic()) {
+ IntrinsicId = Intrinsic::spv_load;
+ Args.push_back(B.getInt32(I.getAlign().value()));
+ } else {
+ IntrinsicId = Intrinsic::spv_atomic_load;
+ Args.push_back(B.getInt8(static_cast<uint8_t>(I.getOrdering())));
+ }
+ CallInst *NewI =
+ B.CreateIntrinsic(IntrinsicId, {I.getOperand(0)->getType()}, Args);
+
replaceMemInstrUses(&I, NewI, B);
return NewI;
}
@@ -2268,11 +2276,18 @@ Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
CB->mutateType(B.getInt32Ty());
}
+ unsigned IntrinsicId;
+ SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
+ B.getInt16(Flags)};
+ if (!I.isAtomic()) {
+ IntrinsicId = Intrinsic::spv_store;
+ Args.push_back(B.getInt32(I.getAlign().value()));
+ } else {
+ IntrinsicId = Intrinsic::spv_atomic_store;
+ Args.push_back(B.getInt8(static_cast<uint8_t>(I.getOrdering())));
+ }
auto *NewI = B.CreateIntrinsic(
- Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
- {I.getValueOperand(), PtrOp, B.getInt16(Flags),
- B.getInt32(I.getAlign().value()),
- B.getInt8(static_cast<uint8_t>(I.getOrdering()))});
+ IntrinsicId, {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
NewI->copyMetadata(I);
I.eraseFromParent();
return NewI;
@@ -2553,7 +2568,9 @@ bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
// Add aliasing decorations to internal load and store intrinsics
// and atomic instructions, skipping atomic store as it won't have ID to
// attach the decoration.
- if (match(Inst, m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>()))
+ if (match(Inst, m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store,
+ Intrinsic::spv_atomic_load,
+ Intrinsic::spv_atomic_store>()))
return true;
auto *CI = dyn_cast<CallInst>(Inst);
if (!CI)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index f6812fd775b6f..5358c1b5bda42 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -109,27 +109,48 @@ void SPIRVTargetLowering::getTgtMemIntrinsic(
SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
MachineFunction &MF, unsigned Intrinsic) const {
IntrinsicInfo Info;
- unsigned AlignIdx = 3;
+
+ unsigned AlignIdx = -1;
+ unsigned OrderingIdx = -1;
+ unsigned FlagsIdx;
+
switch (Intrinsic) {
case Intrinsic::spv_load:
+ FlagsIdx = 1;
AlignIdx = 2;
- [[fallthrough]];
- case Intrinsic::spv_store: {
+ break;
+ case Intrinsic::spv_store:
+ FlagsIdx = 2;
+ AlignIdx = 3;
+ break;
+ case Intrinsic::spv_atomic_load:
+ FlagsIdx = 1;
+ OrderingIdx = 2;
+ break;
+ case Intrinsic::spv_atomic_store:
+ FlagsIdx = 2;
+ OrderingIdx = 3;
+ break;
+ default:
+ return;
+ }
+
+ Info.flags = static_cast<MachineMemOperand::Flags>(
+ cast<ConstantInt>(I.getOperand(FlagsIdx))->getZExtValue());
+ Info.memVT = MVT::i64;
+ // TODO: take into account opaque pointers (don't use getElementType).
+ // MVT::getVT(PtrTy->getElementType());
+
+ if (AlignIdx != -1) {
auto *AlignOp = cast<ConstantInt>(I.getOperand(AlignIdx));
Info.align = Align(AlignOp->getZExtValue());
- Info.flags = static_cast<MachineMemOperand::Flags>(
- cast<ConstantInt>(I.getOperand(AlignIdx - 1))->getZExtValue());
- Info.memVT = MVT::i64;
- // TODO: take into account opaque pointers (don't use getElementType).
- // MVT::getVT(PtrTy->getElementType());
- Info.order = static_cast<AtomicOrdering>(
- cast<ConstantInt>(I.getOperand(AlignIdx + 1))->getZExtValue());
- Infos.push_back(Info);
- return;
}
- default:
- break;
+
+ if (OrderingIdx != -1) {
+ Info.order = static_cast<AtomicOrdering>(
+ cast<ConstantInt>(I.getOperand(OrderingIdx))->getZExtValue());
}
+ Infos.push_back(Info);
}
TargetLowering::ConstraintType
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index ccf3e3e944a8b..e2db4277e3fd9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -160,13 +160,10 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectLoad(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
- bool selectAtomicLoad(Register ResVReg, SPIRVTypeInst ResType, Register Ptr,
- const MachineMemOperand &MemOp,
- MachineIRBuilder &MIRBuilder, MachineInstr &L) const;
+ bool selectAtomicLoad(Register ResVReg, SPIRVTypeInst ResType,
+ MachineInstr &I) const;
bool selectStore(MachineInstr &I) const;
- bool selectAtomicStore(Register StoreVal, Register Ptr,
- const MachineMemOperand &MemOp,
- MachineIRBuilder &MIRBuilder, MachineInstr &S) const;
+ bool selectAtomicStore(MachineInstr &I) const;
bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType,
MachineInstr &I) const;
@@ -1911,7 +1908,7 @@ bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
if (I.getNumMemOperands()) {
const MachineMemOperand *MemOp = *I.memoperands_begin();
if (MemOp->isAtomic())
- return selectAtomicLoad(ResVReg, ResType, Ptr, *MemOp, MIRBuilder, I);
+ return selectAtomicLoad(ResVReg, ResType, I);
}
auto MIB = MIRBuilder.buildInstr(SPIRV::OpLoad)
@@ -1932,30 +1929,33 @@ bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg,
SPIRVTypeInst ResType,
- Register Ptr,
- const MachineMemOperand &MemOp,
- MachineIRBuilder &MIRBuilder,
- MachineInstr &L) const {
- LLVMContext &Context = L.getMF()->getFunction().getContext();
+ MachineInstr &I) const {
+ LLVMContext &Context = I.getMF()->getFunction().getContext();
+
+ unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
+ Register Ptr = I.getOperand(1 + OpOffset).getReg();
if (!ResType.isTypeIntOrFloat())
return diagnoseUnsupported(
- L, "atomic load is only allowed for integer or floating point types");
+ I, "atomic load is only allowed for integer or floating point types");
+ const MachineMemOperand &MemOp = **I.memoperands_begin();
+ assert(MemOp.isAtomic());
// This could be relaxed since the volatile attribute on atomic load is
// supported with the VulkanMemoryModelKHR capability.
if (MemOp.isVolatile())
return diagnoseUnsupported(
- L, "atomic load of volatile memory is not supported");
+ I, "atomic load of volatile memory is not supported");
uint32_t Scope =
static_cast<uint32_t>(getMemScope(Context, MemOp.getSyncScopeID()));
- Register ScopeReg = buildI32Constant(Scope, L);
+ Register ScopeReg = buildI32Constant(Scope, I);
AtomicOrdering AO = MemOp.getSuccessOrdering();
uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
- Register MemSemReg = buildI32Constant(MemSem, L);
+ Register MemSemReg = buildI32Constant(MemSem, I);
+ MachineIRBuilder MIRBuilder(I);
auto AtomicLoad = MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
.addDef(ResVReg)
.addUse(GR.getSPIRVTypeID(ResType))
@@ -2001,14 +2001,13 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
}
}
- MachineIRBuilder MIRBuilder(I);
-
if (I.getNumMemOperands()) {
const MachineMemOperand *MemOp = *I.memoperands_begin();
if (MemOp->isAtomic())
- return selectAtomicStore(StoreVal, Ptr, *MemOp, MIRBuilder, I);
+ return selectAtomicStore(I);
}
+ MachineIRBuilder MIRBuilder(I);
auto MIB = MIRBuilder.buildInstr(SPIRV::OpStore).addUse(Ptr).addUse(StoreVal);
if (!I.getNumMemOperands()) {
assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
@@ -2022,33 +2021,37 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
return true;
}
-bool SPIRVInstructionSelector::selectAtomicStore(Register StoreVal,
- Register Ptr,
- const MachineMemOperand &MemOp,
- MachineIRBuilder &MIRBuilder,
- MachineInstr &S) const {
- LLVMContext &Context = S.getMF()->getFunction().getContext();
+bool SPIRVInstructionSelector::selectAtomicStore(MachineInstr &I) const {
+ LLVMContext &Context = I.getMF()->getFunction().getContext();
+
+ unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
+ Register StoreVal = I.getOperand(0 + OpOffset).getReg();
+ Register Ptr = I.getOperand(1 + OpOffset).getReg();
SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(Ptr);
SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
if (!PointeeType.isTypeIntOrFloat())
return diagnoseUnsupported(
- S, "atomic store is only allowed for integer or floating point types");
+ I, "atomic store is only allowed for integer or floating point types");
+
+ const MachineMemOperand &MemOp = **I.memoperands_begin();
+ assert(MemOp.isAtomic());
// This could be relaxed since the volatile attribute on atomic store is
// supported with the VulkanMemoryModelKHR capability.
if (MemOp.isVolatile())
return diagnoseUnsupported(
- S, "atomic store of volatile memory is not supported");
+ I, "atomic store of volatile memory is not supported");
uint32_t Scope =
static_cast<uint32_t>(getMemScope(Context, MemOp.getSyncScopeID()));
- Register ScopeReg = buildI32Constant(Scope, S);
+ Register ScopeReg = buildI32Constant(Scope, I);
AtomicOrdering AO = MemOp.getSuccessOrdering();
uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
- Register MemSemReg = buildI32Constant(MemSem, S);
+ Register MemSemReg = buildI32Constant(MemSem, I);
+ MachineIRBuilder MIRBuilder(I);
auto AtomicStore = MIRBuilder.buildInstr(SPIRV::OpAtomicStore)
.addUse(Ptr)
.addUse(ScopeReg)
@@ -4540,8 +4543,12 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
switch (IID) {
case Intrinsic::spv_load:
return selectLoad(ResVReg, ResType, I);
+ case Intrinsic::spv_atomic_load:
+ return selectAtomicLoad(ResVReg, ResType, I);
case Intrinsic::spv_store:
return selectStore(I);
+ case Intrinsic::spv_atomic_store:
+ return selectAtomicStore(I);
case Intrinsic::spv_extractv:
return selectExtractVal(ResVReg, ResType, I);
case Intrinsic::spv_insertv:
>From 2107a21c7b8ca9c1fa396af20b12f116c9f8cc3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Tue, 21 Apr 2026 14:52:48 +0200
Subject: [PATCH 4/7] Review atomic load/store start working on the
memory_aliasing info
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 9 ++++-
.../alias-load-store-atomic.ll | 40 +++++++++++++++----
2 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index ef70f57d3b2e2..77467ffb5f2ba 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2569,9 +2569,14 @@ bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
// and atomic instructions, skipping atomic store as it won't have ID to
// attach the decoration.
if (match(Inst, m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store,
- Intrinsic::spv_atomic_load,
- Intrinsic::spv_atomic_store>()))
+ Intrinsic::spv_atomic_load>()))
return true;
+
+ if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
+ if (L->isAtomic())
+ return true;
+ }
+
auto *CI = dyn_cast<CallInst>(Inst);
if (!CI)
return false;
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll
index 96b77ef52e42e..6ad1a60b78836 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll
@@ -1,4 +1,5 @@
-; Check aliasing information translation on atomic load and store
+; Check aliasing information translation on atomic load and store.
+; For stores, the decoration is not generated since the store opcodes do not have an id.
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown -verify-machineinstrs --spirv-ext=+SPV_INTEL_memory_access_aliasing %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_memory_access_aliasing %s -o - -filetype=obj | spirv-val %}
@@ -8,23 +9,42 @@
; CHECK: %[[#Domain1:]] = OpAliasDomainDeclINTEL
; CHECK: %[[#Scope1:]] = OpAliasScopeDeclINTEL %[[#Domain1]]
; CHECK: %[[#List1:]] = OpAliasScopeListDeclINTEL %[[#Scope1]]
-; CHECK: OpDecorateId %[[#Load:]] NoAliasINTEL %[[#List1]]
-; CHECK: %[[#Load:]] = OpAtomicLoad
+; CHECK: %[[#Domain2:]] = OpAliasDomainDeclINTEL
+; CHECK: %[[#Scope2:]] = OpAliasScopeDeclINTEL %[[#Domain2]]
+; CHECK: %[[#List2:]] = OpAliasScopeListDeclINTEL %[[#Scope2]]
+; CHECK: OpDecorateId %[[#LoadFun:]] NoAliasINTEL %[[#List1]]
+; CHECK: OpDecorateId %[[#LoadInst:]] NoAliasINTEL %[[#List2]]
+; CHECK-NOT: OpDecorateId
+; CHECK: %[[#LoadFun:]] = OpAtomicLoad
+; CHECK: OpAtomicStore
+; CHECK: %[[#LoadInst:]] = OpAtomicLoad
+; CHECK: OpAtomicStore
-define spir_func i32 @test_load(ptr addrspace(4) %object) #0 {
+define spir_func i32 @test_load_call(ptr addrspace(4) %object) #0 {
entry:
%0 = call spir_func i32 @_Z18__spirv_AtomicLoadPU3AS4iii(ptr addrspace(4) %object, i32 1, i32 16), !noalias !1
ret i32 %0
}
-declare spir_func i32 @_Z18__spirv_AtomicLoadPU3AS4iii(ptr addrspace(4), i32, i32)
-
-define spir_func void @test_store(ptr addrspace(4) %object, ptr addrspace(4) %expected, i32 %desired) {
+define spir_func void @test_store_call(ptr addrspace(4) %object, i32 %desired) {
entry:
call spir_func void @_Z19__spirv_AtomicStorePU3AS4iiii(ptr addrspace(4) %object, i32 1, i32 16, i32 %desired), !noalias !4
ret void
}
+define spir_func i32 @test_load_instr(ptr addrspace(4) %object) #0 {
+entry:
+ %0 = load atomic i32, ptr addrspace(4) %object syncscope("singlethread") acquire, align 4, !noalias !7
+ ret i32 %0
+}
+
+define spir_func void @test_store_instr(ptr addrspace(4) %object, i32 %desired) {
+entry:
+ store atomic i32 %desired, ptr addrspace(4) %object syncscope("singlethread") release, align 4, !noalias !10
+ ret void
+}
+
+declare spir_func i32 @_Z18__spirv_AtomicLoadPU3AS4iii(ptr addrspace(4), i32, i32)
declare spir_func void @_Z19__spirv_AtomicStorePU3AS4iiii(ptr addrspace(4), i32, i32, i32)
!1 = !{!2}
@@ -33,3 +53,9 @@ declare spir_func void @_Z19__spirv_AtomicStorePU3AS4iiii(ptr addrspace(4), i32,
!4 = !{!5}
!5 = distinct !{!5, !6}
!6 = distinct !{!6}
+!7 = !{!8}
+!8 = distinct !{!8, !9}
+!9 = distinct !{!9}
+!10 = !{!11}
+!11 = distinct !{!11, !12}
+!12 = distinct !{!12}
>From 739affbafd58755e0818f17add7b9104c5ea38b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Wed, 22 Apr 2026 10:58:38 +0200
Subject: [PATCH 5/7] Review: mrsidims remarks
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 22 ++++++++++---------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index e2db4277e3fd9..1c3c13f01689a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1936,16 +1936,17 @@ bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg,
Register Ptr = I.getOperand(1 + OpOffset).getReg();
if (!ResType.isTypeIntOrFloat())
- return diagnoseUnsupported(
- I, "atomic load is only allowed for integer or floating point types");
+ return diagnoseUnsupported(I,
+ "Lowering to SPIR-V of atomic load is only "
+ "allowed for integer or floating point types");
const MachineMemOperand &MemOp = **I.memoperands_begin();
assert(MemOp.isAtomic());
- // This could be relaxed since the volatile attribute on atomic load is
+ // TODO: This must be relaxed since the volatile attribute on atomic load is
// supported with the VulkanMemoryModelKHR capability.
if (MemOp.isVolatile())
- return diagnoseUnsupported(
- I, "atomic load of volatile memory is not supported");
+ return diagnoseUnsupported(I, "Lowering to SPIR-V of atomic load of "
+ "volatile memory is not supported");
uint32_t Scope =
static_cast<uint32_t>(getMemScope(Context, MemOp.getSyncScopeID()));
@@ -2031,17 +2032,18 @@ bool SPIRVInstructionSelector::selectAtomicStore(MachineInstr &I) const {
SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(Ptr);
SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
if (!PointeeType.isTypeIntOrFloat())
- return diagnoseUnsupported(
- I, "atomic store is only allowed for integer or floating point types");
+ return diagnoseUnsupported(I,
+ "Lowering to SPIR-V of atomic store is only "
+ "allowed for integer or floating point types");
const MachineMemOperand &MemOp = **I.memoperands_begin();
assert(MemOp.isAtomic());
- // This could be relaxed since the volatile attribute on atomic store is
+ // TODO: This must be relaxed since the volatile attribute on atomic store is
// supported with the VulkanMemoryModelKHR capability.
if (MemOp.isVolatile())
- return diagnoseUnsupported(
- I, "atomic store of volatile memory is not supported");
+ return diagnoseUnsupported(I, "Lowering to SPIR-V of atomic store of "
+ "volatile memory is not supported");
uint32_t Scope =
static_cast<uint32_t>(getMemScope(Context, MemOp.getSyncScopeID()));
>From ad08ffe602207c84abb960e32f982e249397379a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Thu, 23 Apr 2026 10:48:41 +0200
Subject: [PATCH 6/7] Review: Change of course, do not emit
spv_intel_memory_access_aliasing for atomic load/store
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 8 +-------
.../alias-load-store-atomic.ll | 5 +----
2 files changed, 2 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 77467ffb5f2ba..c7d506b60391d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2568,15 +2568,9 @@ bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
// Add aliasing decorations to internal load and store intrinsics
// and atomic instructions, skipping atomic store as it won't have ID to
// attach the decoration.
- if (match(Inst, m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store,
- Intrinsic::spv_atomic_load>()))
+ if (match(Inst, m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>()))
return true;
- if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
- if (L->isAtomic())
- return true;
- }
-
auto *CI = dyn_cast<CallInst>(Inst);
if (!CI)
return false;
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll
index 6ad1a60b78836..5abde1a9fec66 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-load-store-atomic.ll
@@ -1,5 +1,6 @@
; Check aliasing information translation on atomic load and store.
; For stores, the decoration is not generated since the store opcodes do not have an id.
+; For `load atomic`, the decoration is not generated because we're dropping the extension on atomics.
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown -verify-machineinstrs --spirv-ext=+SPV_INTEL_memory_access_aliasing %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_memory_access_aliasing %s -o - -filetype=obj | spirv-val %}
@@ -9,11 +10,7 @@
; CHECK: %[[#Domain1:]] = OpAliasDomainDeclINTEL
; CHECK: %[[#Scope1:]] = OpAliasScopeDeclINTEL %[[#Domain1]]
; CHECK: %[[#List1:]] = OpAliasScopeListDeclINTEL %[[#Scope1]]
-; CHECK: %[[#Domain2:]] = OpAliasDomainDeclINTEL
-; CHECK: %[[#Scope2:]] = OpAliasScopeDeclINTEL %[[#Domain2]]
-; CHECK: %[[#List2:]] = OpAliasScopeListDeclINTEL %[[#Scope2]]
; CHECK: OpDecorateId %[[#LoadFun:]] NoAliasINTEL %[[#List1]]
-; CHECK: OpDecorateId %[[#LoadInst:]] NoAliasINTEL %[[#List2]]
; CHECK-NOT: OpDecorateId
; CHECK: %[[#LoadFun:]] = OpAtomicLoad
; CHECK: OpAtomicStore
>From c178412a8b93a7bacd22138addf26586b62b36d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
<jmartinezcaamao at gmail.com>
Date: Thu, 23 Apr 2026 10:59:09 +0200
Subject: [PATCH 7/7] Update llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
Co-authored-by: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 1c3c13f01689a..8efb99ede7c32 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1940,6 +1940,7 @@ bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg,
"Lowering to SPIR-V of atomic load is only "
"allowed for integer or floating point types");
+ assert(I.getNumMemOperands());
const MachineMemOperand &MemOp = **I.memoperands_begin();
assert(MemOp.isAtomic());
// TODO: This must be relaxed since the volatile attribute on atomic load is
More information about the llvm-commits
mailing list