[llvm] [SPIRV] Translate {uinc/udec}_wrap as opaque intrinsics for AMD (PR #215507)
Marcos Maronas via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 13:00:50 PDT 2026
https://github.com/maarquitos14 updated https://github.com/llvm/llvm-project/pull/215507
>From 98dbb4191e5154116659ee1e899d624ded3749d9 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 04:33:00 -0500
Subject: [PATCH 1/3] [SPIRV] Translate atomicrmw {uinc,udec}_wrap as
OpFunctionCall
atomicrmw uinc_wrap/udec_wrap had no direct SPIR-V representation and were
expanded into a compare-exchange loop before reaching the SPIRV backend,
which prevents targets with native wrapping increment/decrement atomics
(e.g. AMDGPU ds_inc_u32/ds_dec_u32) from recovering the original operation.
Keep the atomicrmw intact by returning AtomicExpansionKind::None from
shouldExpandAtomicRMWInIR, and translate it in SPIRVEmitIntrinsics to a
SPIR_FUNC call to __translate_spirv_atomic_uinc_wrap /
__translate_spirv_atomic_udec_wrap, with scope and memory semantics derived
from the instruction's syncscope and ordering. The imported declaration is
emitted with LinkageAttributes Import. All other atomicrmw operations are
unaffected.
The helpers deliberately avoid the __spirv_ prefix: that namespace is
reserved for SPIR-V friendly IR, where a name maps to a SPIR-V opcode (see
__spirv_AtomicUMin in SPIRVBuiltins.td). No such opcode exists for these
operations, so the __translate_ prefix already used for similar
translation-only symbols is used instead.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 56 ++++++++++++++++
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 5 ++
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 2 +
llvm/lib/Target/SPIRV/SPIRVUtils.h | 3 +
.../SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll | 30 +++++++++
.../atomicrmw-uinc-udec-wrap-orderings.ll | 66 ++++++++++++++++++
.../SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll | 59 ++++++++++++++++
.../atomicrmw-uinc-udec-wrap-signatures.ll | 67 +++++++++++++++++++
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 55 ++++++---------
9 files changed, 308 insertions(+), 35 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 8d0b80b9a2e1c..a0449e7a6f5f6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -394,6 +394,7 @@ class SPIRVEmitIntrinsicsImpl
Instruction *visitStoreInst(StoreInst &I);
Instruction *visitAllocaInst(AllocaInst &I);
Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
Instruction *visitUnreachableInst(UnreachableInst &I);
Instruction *visitCallInst(CallInst &I);
@@ -2620,6 +2621,61 @@ SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
return NewI;
}
+Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
+ auto Op = I.getOperation();
+ if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
+ return &I;
+
+ // Carrying these across the SPIR-V boundary as a call to an imported helper
+ // is an AMD extension: there is no SPIR-V opcode for them, so a consumer has
+ // to recognize the helper by name to make sense of the module. Restrict it to
+ // AMD targets and let everyone else keep the generic expansion.
+ if (!isAMDTarget(TM.getTargetTriple()))
+ return &I;
+
+ Module *M = I.getModule();
+ IRBuilder<> B(I.getParent());
+ B.SetInsertPoint(&I);
+
+ const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(*I.getFunction());
+ unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
+
+ uint32_t Scope = static_cast<uint32_t>(
+ getMemScope(TM.getTargetTriple(), I.getContext(), I.getSyncScopeID()));
+ uint32_t ScSem = static_cast<uint32_t>(
+ getMemSemanticsForStorageClass(addressSpaceToStorageClass(AS, ST)));
+ uint32_t MemSem =
+ static_cast<uint32_t>(getMemSemantics(I.getOrdering())) | ScSem;
+
+ std::string FuncName = (Op == AtomicRMWInst::UIncWrap)
+ ? "__translate_spirv_atomic_uinc_wrap"
+ : "__translate_spirv_atomic_udec_wrap";
+
+ Type *ValTy = I.getValOperand()->getType();
+ Type *PtrTy = I.getPointerOperand()->getType();
+ // Encode the address space and the value type in the name, the same way
+ // lowerLLVMIntrinsicName() does for spirv.llvm_memset_p1_i64. A module may
+ // need several mutually incompatible signatures, while SPIR-V resolves an
+ // imported function by its linkage name alone.
+ FuncName += "_p" + std::to_string(AS) + "_i" +
+ std::to_string(ValTy->getIntegerBitWidth());
+
+ Type *Int32Ty = B.getInt32Ty();
+ SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
+ FunctionType *FT = FunctionType::get(ValTy, ArgTys, false);
+ FunctionCallee FC = M->getOrInsertFunction(FuncName, FT);
+ if (auto *F = dyn_cast<Function>(FC.getCallee()))
+ F->setCallingConv(CallingConv::SPIR_FUNC);
+
+ SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt32(Scope),
+ B.getInt32(MemSem), I.getValOperand()};
+ CallInst *CI = B.CreateCall(FC, Args);
+ CI->setCallingConv(CallingConv::SPIR_FUNC);
+
+ replaceAllUsesWithAndErase(B, &I, CI);
+ return CI;
+}
+
static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
auto *CI = dyn_cast<CallInst>(&I);
if (!CI)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 245d2745f1498..674d9f3e5eabf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -679,6 +679,11 @@ SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
return AtomicExpansionKind::None;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ // On AMD targets these are translated into a call to an imported helper by
+ // SPIRVEmitIntrinsics, so they must survive to that point unexpanded. Any
+ // other target has no such helper and needs the generic expansion.
+ return isAMDTarget(STI.getTargetTriple()) ? AtomicExpansionKind::None
+ : AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Nand:
return AtomicExpansionKind::CmpXChg;
default:
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index ce993299096ad..321a324d19e07 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -449,6 +449,8 @@ SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
llvm_unreachable(nullptr);
}
+bool isAMDTarget(const Triple &T) { return T.getVendor() == Triple::AMD; }
+
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
SyncScope::ID Id) {
// Named by
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index b95f09eba95f1..262e41674f531 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -289,6 +289,9 @@ SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord);
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
SyncScope::ID Id);
+// Returns true if T targets an AMD SPIR-V flavour.
+bool isAMDTarget(const Triple &T);
+
// Find def instruction for the given ConstReg, walking through
// spv_track_constant and ASSIGN_TYPE instructions. Updates ConstReg by def
// of OpConstant instruction.
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
new file mode 100644
index 0000000000000..fde6dd15cc0d5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
@@ -0,0 +1,30 @@
+; Translating atomicrmw uinc_wrap/udec_wrap into a call to an imported helper is
+; an AMD extension: there is no SPIR-V opcode for these, so a consumer has to
+; recognize the helper by name to make sense of the module. Verify that a
+; non-AMD target does not emit it, and instead falls back to the generic CmpXChg
+; expansion. The AMD behaviour is covered by atomicrmw-uinc-udec-wrap.ll.
+;
+; --implicit-check-not applies over the whole module, unlike a CHECK-NOT, which
+; would only cover the input up to the first positive match below.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --implicit-check-not=__translate_spirv_atomic
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --implicit-check-not=__translate_spirv_atomic
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+; Both operations expand to an OpAtomicCompareExchange retry loop.
+; CHECK: OpAtomicCompareExchange
+define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
+entry:
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+ ret void
+}
+
+; CHECK: OpAtomicCompareExchange
+define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
+entry:
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
new file mode 100644
index 0000000000000..80c771b31eb54
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
@@ -0,0 +1,66 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode memory
+; orderings in the function call arguments.
+; CrossWorkgroupMemory = 0x200 = 512
+; Monotonic (Relaxed) = 0x000 -> with CrossWorkgroup: 512
+; Acquire = 0x002 -> with CrossWorkgroup: 514
+; Release = 0x004 -> with CrossWorkgroup: 516
+; AcquireRelease = 0x008 -> with CrossWorkgroup: 520
+; SequentiallyConsistent = 0x010 -> with CrossWorkgroup: 528
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#MemSem_Monotonic:]] = OpConstant %[[#Int]] 512
+; CHECK-DAG: %[[#MemSem_Acquire:]] = OpConstant %[[#Int]] 514
+; CHECK-DAG: %[[#MemSem_Release:]] = OpConstant %[[#Int]] 516
+; CHECK-DAG: %[[#MemSem_AcqRel:]] = OpConstant %[[#Int]] 520
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
new file mode 100644
index 0000000000000..c748e47807f63
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
@@ -0,0 +1,59 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode scopes.
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#Scope_Device:]] = OpConstant %[[#Int]] 1{{$}}
+; CHECK-DAG: %[[#Scope_Workgroup:]] = OpConstant %[[#Int]] 2{{$}}
+; CHECK-DAG: %[[#Scope_Subgroup:]] = OpConstant %[[#Int]] 3{{$}}
+; CHECK-DAG: %[[#Scope_Invocation:]] = OpConstant %[[#Int]] 4{{$}}
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528{{$}}
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
new file mode 100644
index 0000000000000..f6fd4cf3ed6ab
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
@@ -0,0 +1,67 @@
+; A module may contain uinc_wrap/udec_wrap atomics with mutually incompatible
+; signatures, while SPIR-V resolves an imported function by its linkage name
+; alone. Verify that the _p<addrspace>_i<width> suffix keeps them apart: each
+; distinct (address space, value type) combination gets its own declaration,
+; and every call site of a given combination shares that one declaration.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
+; CHECK-DAG: %[[#Value64:]] = OpConstant %[[#Long]] 42
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; The storage class contributes to the memory semantics, so an atomic on a
+; Workgroup pointer carries WorkgroupMemory (256) where a CrossWorkgroup one
+; carries CrossWorkgroupMemory (512).
+; CHECK-DAG: %[[#MemSem_Relaxed_Local:]] = OpConstant %[[#Int]] 256
+; CHECK-DAG: %[[#MemSem_Relaxed:]] = OpConstant %[[#Int]] 512
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UIncFnLocal:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p3_i32" Import
+; CHECK-DAG: OpDecorate %[[#UIncFn64:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i64" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+ at lui = common dso_local addrspace(3) global i32 0, align 4
+ at ul = common dso_local addrspace(1) global i64 0, align 8
+
+; Two atomics of the same value type in different address spaces need two
+; incompatible signatures, so they must resolve to two distinct declarations.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFnLocal]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed_Local]] %[[#Value]]
+define dso_local spir_func void @mixed_addrspace() local_unnamed_addr {
+entry:
+ %g = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %l = atomicrmw uinc_wrap ptr addrspace(3) @lui, i32 42 monotonic
+ ret void
+}
+
+; Likewise for two atomics in the same address space with different value
+; widths: the i64 one must not reuse the i32 declaration, and its call must
+; yield an i64 result.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Long]] %[[#UIncFn64]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value64]]
+define dso_local spir_func void @mixed_width() local_unnamed_addr {
+entry:
+ %a = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %b = atomicrmw uinc_wrap ptr addrspace(1) @ul, i64 42 monotonic
+ ret void
+}
+
+; uinc_wrap and udec_wrap on the same (address space, value type) are still two
+; separate symbols, and repeated call sites reuse the single declaration.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+define dso_local spir_func void @shared_declaration() local_unnamed_addr {
+entry:
+ %a = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %b = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
index ed53ea525e39e..66c01ef739a60 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -1,56 +1,41 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; Verify that on AMD targets atomicrmw uinc_wrap/udec_wrap lower to
+; OpFunctionCall to
+; __translate_spirv_atomic_uinc_wrap_*/__translate_spirv_atomic_udec_wrap_* with
+; Import linkage, rather than being expanded to a CmpXChg loop. The name carries
+; a _p<addrspace>_i<width> suffix, because a module may need several mutually
+; incompatible signatures while SPIR-V resolves an imported function by its
+; linkage name alone.
+;
+; The helper is an AMD extension, so non-AMD targets keep the generic expansion
+; instead; that is covered by atomicrmw-uinc-udec-wrap-non-amd.ll.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Bool:]] = OpTypeBool
; CHECK-DAG: %[[#PointerType:]] = OpTypePointer CrossWorkgroup %[[#Int]]
; CHECK-DAG: %[[#MemSem_SequentiallyConsistent:]] = OpConstant %[[#Int]] 528
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
-; CHECK-DAG: %[[#One:]] = OpConstant %[[#Int]] 1
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#PointerType]] CrossWorkgroup
; CHECK-DAG: %[[#AllOnes:]] = OpConstant %[[#Int]] 4294967295
- at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK-DAG: OpDecorate %[[#UIncWrapFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecWrapFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Add:]] = OpIAdd %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#GE:]] = OpUGreaterThanEqual %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#GE]] %[[#Scope_CrossDevice]] %[[#Add]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SequentiallyConsistent]] %[[#Value]]
define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
ret void
}
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Sub:]] = OpISub %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#Equal:]] = OpIEqual %[[#Bool]] %[[#Phi]] %[[#Scope_CrossDevice]]
-; CHECK: %[[#GT:]] = OpUGreaterThan %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Or:]] = OpLogicalOr %[[#Bool]] %[[#Equal]] %[[#GT]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#Or]] %[[#Value]] %[[#Sub]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
-
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SequentiallyConsistent]] %[[#Value]]
define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
>From e66d679f88d361af3421e90e07c13ffa25d43186 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Tue, 11 Aug 2026 10:27:21 -0500
Subject: [PATCH 2/3] Add support for vector types and corresponding tests.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 12 +++++++---
.../atomicrmw-uinc-udec-wrap-oversized.ll | 22 +++++++++++++++++++
.../atomicrmw-uinc-udec-wrap-signatures.ll | 21 ++++++++++++++++--
3 files changed, 50 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index a0449e7a6f5f6..eab8891a8c6d8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2656,9 +2656,15 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
// Encode the address space and the value type in the name, the same way
// lowerLLVMIntrinsicName() does for spirv.llvm_memset_p1_i64. A module may
// need several mutually incompatible signatures, while SPIR-V resolves an
- // imported function by its linkage name alone.
- FuncName += "_p" + std::to_string(AS) + "_i" +
- std::to_string(ValTy->getIntegerBitWidth());
+ // imported function by its linkage name alone. The value may also be a fixed
+ // vector of integers, spelled the LLVM way: _p1_v2i32. Anything wider than
+ // the target's maximum atomic size never reaches here, because AtomicExpand
+ // rejects it first.
+ std::string TypeSuffix;
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ValTy))
+ TypeSuffix = "v" + std::to_string(VecTy->getNumElements());
+ TypeSuffix += "i" + std::to_string(ValTy->getScalarSizeInBits());
+ FuncName += "_p" + std::to_string(AS) + "_" + TypeSuffix;
Type *Int32Ty = B.getInt32Ty();
SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll
new file mode 100644
index 0000000000000..d258bd7d829ba
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll
@@ -0,0 +1,22 @@
+; The SPIR-V target supports no atomic wider than 64 bits
+; (SPIRVTargetLowering sets setMaxAtomicSizeInBitsSupported(64)), so a vector
+; operand that does not fit is rejected by AtomicExpandPass before the
+; uinc_wrap/udec_wrap helper lowering ever sees it. Verify that carrying these
+; two operations across as an imported helper on AMD targets does not smuggle
+; an over-limit atomic past that check: the diagnostic is the same one any
+; other target and any other atomicrmw operation gets. The supported vector
+; widths are covered by atomicrmw-uinc-udec-wrap-signatures.ll.
+
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - 2>&1 | FileCheck %s
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - 2>&1 | FileCheck %s
+
+; CHECK: error: unsupported atomicrmw uinc_wrap: target supports atomics up to 8 bytes, but this atomic accesses 16 bytes
+; CHECK-NOT: __translate_spirv_atomic
+
+ at uv = common dso_local addrspace(1) global <4 x i32> zeroinitializer, align 16
+
+define dso_local spir_func void @oversized_vector() local_unnamed_addr {
+entry:
+ %v = atomicrmw uinc_wrap ptr addrspace(1) @uv, <4 x i32> splat (i32 42) monotonic
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
index f6fd4cf3ed6ab..04e894f0d2263 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
@@ -1,8 +1,9 @@
; A module may contain uinc_wrap/udec_wrap atomics with mutually incompatible
; signatures, while SPIR-V resolves an imported function by its linkage name
-; alone. Verify that the _p<addrspace>_i<width> suffix keeps them apart: each
+; alone. Verify that the _p<addrspace>_<type> suffix keeps them apart: each
; distinct (address space, value type) combination gets its own declaration,
-; and every call site of a given combination shares that one declaration.
+; and every call site of a given combination shares that one declaration. The
+; value type may be a fixed vector of integers, spelled the LLVM way: v2i32.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
@@ -11,8 +12,10 @@
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#Vec:]] = OpTypeVector %[[#Int]] 2
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
; CHECK-DAG: %[[#Value64:]] = OpConstant %[[#Long]] 42
+; CHECK-DAG: %[[#ValueVec:]] = OpConstantComposite %[[#Vec]] %[[#Value]] %[[#Value]]
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
; The storage class contributes to the memory semantics, so an atomic on a
; Workgroup pointer carries WorkgroupMemory (256) where a CrossWorkgroup one
@@ -23,11 +26,13 @@
; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
; CHECK-DAG: OpDecorate %[[#UIncFnLocal:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p3_i32" Import
; CHECK-DAG: OpDecorate %[[#UIncFn64:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i64" Import
+; CHECK-DAG: OpDecorate %[[#UIncFnVec:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_v2i32" Import
; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
@ui = common dso_local addrspace(1) global i32 0, align 4
@lui = common dso_local addrspace(3) global i32 0, align 4
@ul = common dso_local addrspace(1) global i64 0, align 8
+ at uv = common dso_local addrspace(1) global <2 x i32> zeroinitializer, align 8
; Two atomics of the same value type in different address spaces need two
; incompatible signatures, so they must resolve to two distinct declarations.
@@ -65,3 +70,15 @@ entry:
%b = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
ret void
}
+
+; A vector operand is carried across just like a scalar one, under its own
+; symbol, as long as it fits in the target's maximum atomic size. Anything
+; wider is rejected by AtomicExpandPass before this lowering runs; see
+; atomicrmw-uinc-udec-wrap-oversized.ll.
+
+; CHECK: OpFunctionCall %[[#Vec]] %[[#UIncFnVec]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#ValueVec]]
+define dso_local spir_func void @vector_value() local_unnamed_addr {
+entry:
+ %v = atomicrmw uinc_wrap ptr addrspace(1) @uv, <2 x i32> splat (i32 42) monotonic
+ ret void
+}
>From 73925063228ad2506ae558295d73d89c7066335a Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Tue, 11 Aug 2026 13:16:07 -0500
Subject: [PATCH 3/3] [SPIRV] Keep the metadata on the lowered
uinc_wrap/udec_wrap call
The other memory instructions lowered by this pass copy the metadata of
the instruction they replace. SPIRVCallLowering reads alias.scope and
noalias off the call to build the aliasing decorations and runs after
this pass, so dropping them here loses them silently.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index eab8891a8c6d8..25ea9b7823b64 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2677,6 +2677,12 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
B.getInt32(MemSem), I.getValOperand()};
CallInst *CI = B.CreateCall(FC, Args);
CI->setCallingConv(CallingConv::SPIR_FUNC);
+ // Keep the metadata, as the other memory instructions lowered here do.
+ // SPIRVCallLowering reads alias.scope/noalias off the call to build the
+ // aliasing decorations, and it runs after this pass, so dropping them here
+ // would silently lose them. insertSpirvDecorations() has already run for the
+ // atomicrmw and does not revisit the call, so nothing is encoded twice.
+ CI->copyMetadata(I);
replaceAllUsesWithAndErase(B, &I, CI);
return CI;
More information about the llvm-commits
mailing list