[llvm] [SPIRV] Preserve AMDGPU metadata through NonSemantic.AuxData (PR #215510)
Marcos Maronas via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 03:22:07 PDT 2026
https://github.com/maarquitos14 created https://github.com/llvm/llvm-project/pull/215510
The PR implements an extension to `NonSemantic.AuxData` spec to add `NonSemanticAuxDataInstructionMetadata`, which enables preservation of metadata for instructions other than `GlobalVariable` and `Function` objects for some AMDGPU metadata.
Part 2 of #213685 split, stacked on top of #215507.
>From 147ad9d42be874dd7bd3121f8cb7d4667a342454 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 04:33:00 -0500
Subject: [PATCH 1/4] [SPIRV] Translate atomicrmw {uinc,udec}_wrap as
OpFunctionCall
atomicrmw uinc_wrap/udec_wrap had no direct SPIR-V representation and were
expanded into a compare-exchange loop before reaching the SPIRV backend,
which prevents targets with native wrapping increment/decrement atomics
(e.g. AMDGPU ds_inc_u32/ds_dec_u32) from recovering the original operation.
Keep the atomicrmw intact by returning AtomicExpansionKind::None from
shouldExpandAtomicRMWInIR, and translate it in SPIRVEmitIntrinsics to a
SPIR_FUNC call to __translate_spirv_atomic_uinc_wrap /
__translate_spirv_atomic_udec_wrap, with scope and memory semantics derived
from the instruction's syncscope and ordering. The imported declaration is
emitted with LinkageAttributes Import. All other atomicrmw operations are
unaffected.
The helpers deliberately avoid the __spirv_ prefix: that namespace is
reserved for SPIR-V friendly IR, where a name maps to a SPIR-V opcode (see
__spirv_AtomicUMin in SPIRVBuiltins.td). No such opcode exists for these
operations, so the __translate_ prefix already used for similar
translation-only symbols is used instead.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 56 ++++++++++++++++
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 5 ++
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 2 +
llvm/lib/Target/SPIRV/SPIRVUtils.h | 8 +++
.../SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll | 30 +++++++++
.../atomicrmw-uinc-udec-wrap-orderings.ll | 66 ++++++++++++++++++
.../SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll | 59 ++++++++++++++++
.../atomicrmw-uinc-udec-wrap-signatures.ll | 67 +++++++++++++++++++
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 55 ++++++---------
9 files changed, 313 insertions(+), 35 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 8d0b80b9a2e1c..a0449e7a6f5f6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -394,6 +394,7 @@ class SPIRVEmitIntrinsicsImpl
Instruction *visitStoreInst(StoreInst &I);
Instruction *visitAllocaInst(AllocaInst &I);
Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
Instruction *visitUnreachableInst(UnreachableInst &I);
Instruction *visitCallInst(CallInst &I);
@@ -2620,6 +2621,61 @@ SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
return NewI;
}
+Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
+ auto Op = I.getOperation();
+ if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
+ return &I;
+
+ // Carrying these across the SPIR-V boundary as a call to an imported helper
+ // is an AMD extension: there is no SPIR-V opcode for them, so a consumer has
+ // to recognize the helper by name to make sense of the module. Restrict it to
+ // AMD targets and let everyone else keep the generic expansion.
+ if (!isAMDTarget(TM.getTargetTriple()))
+ return &I;
+
+ Module *M = I.getModule();
+ IRBuilder<> B(I.getParent());
+ B.SetInsertPoint(&I);
+
+ const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(*I.getFunction());
+ unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
+
+ uint32_t Scope = static_cast<uint32_t>(
+ getMemScope(TM.getTargetTriple(), I.getContext(), I.getSyncScopeID()));
+ uint32_t ScSem = static_cast<uint32_t>(
+ getMemSemanticsForStorageClass(addressSpaceToStorageClass(AS, ST)));
+ uint32_t MemSem =
+ static_cast<uint32_t>(getMemSemantics(I.getOrdering())) | ScSem;
+
+ std::string FuncName = (Op == AtomicRMWInst::UIncWrap)
+ ? "__translate_spirv_atomic_uinc_wrap"
+ : "__translate_spirv_atomic_udec_wrap";
+
+ Type *ValTy = I.getValOperand()->getType();
+ Type *PtrTy = I.getPointerOperand()->getType();
+ // Encode the address space and the value type in the name, the same way
+ // lowerLLVMIntrinsicName() does for spirv.llvm_memset_p1_i64. A module may
+ // need several mutually incompatible signatures, while SPIR-V resolves an
+ // imported function by its linkage name alone.
+ FuncName += "_p" + std::to_string(AS) + "_i" +
+ std::to_string(ValTy->getIntegerBitWidth());
+
+ Type *Int32Ty = B.getInt32Ty();
+ SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
+ FunctionType *FT = FunctionType::get(ValTy, ArgTys, false);
+ FunctionCallee FC = M->getOrInsertFunction(FuncName, FT);
+ if (auto *F = dyn_cast<Function>(FC.getCallee()))
+ F->setCallingConv(CallingConv::SPIR_FUNC);
+
+ SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt32(Scope),
+ B.getInt32(MemSem), I.getValOperand()};
+ CallInst *CI = B.CreateCall(FC, Args);
+ CI->setCallingConv(CallingConv::SPIR_FUNC);
+
+ replaceAllUsesWithAndErase(B, &I, CI);
+ return CI;
+}
+
static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
auto *CI = dyn_cast<CallInst>(&I);
if (!CI)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 245d2745f1498..674d9f3e5eabf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -679,6 +679,11 @@ SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
return AtomicExpansionKind::None;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ // On AMD targets these are translated into a call to an imported helper by
+ // SPIRVEmitIntrinsics, so they must survive to that point unexpanded. Any
+ // other target has no such helper and needs the generic expansion.
+ return isAMDTarget(STI.getTargetTriple()) ? AtomicExpansionKind::None
+ : AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Nand:
return AtomicExpansionKind::CmpXChg;
default:
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index ce993299096ad..e1384e900124f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -449,6 +449,8 @@ SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
llvm_unreachable(nullptr);
}
+bool isAMDTarget(const Triple &TT) { return TT.getVendor() == Triple::AMD; }
+
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
SyncScope::ID Id) {
// Named by
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index b95f09eba95f1..fe18ce47c54a2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -289,6 +289,14 @@ SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord);
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
SyncScope::ID Id);
+// Returns true if TT targets an AMD SPIR-V flavour. Gates AMD-specific
+// extensions to the emitted SPIR-V that a generic consumer could not process,
+// such as translating atomicrmw uinc_wrap/udec_wrap into a call to an imported
+// helper. Both the decision to lower and the decision not to expand such an
+// atomicrmw generically must consult this, or the operation reaches the
+// legalizer neither expanded nor lowered.
+bool isAMDTarget(const Triple &TT);
+
// Find def instruction for the given ConstReg, walking through
// spv_track_constant and ASSIGN_TYPE instructions. Updates ConstReg by def
// of OpConstant instruction.
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
new file mode 100644
index 0000000000000..fde6dd15cc0d5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
@@ -0,0 +1,30 @@
+; Translating atomicrmw uinc_wrap/udec_wrap into a call to an imported helper is
+; an AMD extension: there is no SPIR-V opcode for these, so a consumer has to
+; recognize the helper by name to make sense of the module. Verify that a
+; non-AMD target does not emit it, and instead falls back to the generic CmpXChg
+; expansion. The AMD behaviour is covered by atomicrmw-uinc-udec-wrap.ll.
+;
+; --implicit-check-not applies over the whole module, unlike a CHECK-NOT, which
+; would only cover the input up to the first positive match below.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --implicit-check-not=__translate_spirv_atomic
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --implicit-check-not=__translate_spirv_atomic
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+; Both operations expand to an OpAtomicCompareExchange retry loop.
+; CHECK: OpAtomicCompareExchange
+define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
+entry:
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+ ret void
+}
+
+; CHECK: OpAtomicCompareExchange
+define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
+entry:
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
new file mode 100644
index 0000000000000..80c771b31eb54
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
@@ -0,0 +1,66 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode memory
+; orderings in the function call arguments.
+; CrossWorkgroupMemory = 0x200 = 512
+; Monotonic (Relaxed) = 0x000 -> with CrossWorkgroup: 512
+; Acquire = 0x002 -> with CrossWorkgroup: 514
+; Release = 0x004 -> with CrossWorkgroup: 516
+; AcquireRelease = 0x008 -> with CrossWorkgroup: 520
+; SequentiallyConsistent = 0x010 -> with CrossWorkgroup: 528
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#MemSem_Monotonic:]] = OpConstant %[[#Int]] 512
+; CHECK-DAG: %[[#MemSem_Acquire:]] = OpConstant %[[#Int]] 514
+; CHECK-DAG: %[[#MemSem_Release:]] = OpConstant %[[#Int]] 516
+; CHECK-DAG: %[[#MemSem_AcqRel:]] = OpConstant %[[#Int]] 520
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
new file mode 100644
index 0000000000000..c748e47807f63
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
@@ -0,0 +1,59 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode scopes.
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#Scope_Device:]] = OpConstant %[[#Int]] 1{{$}}
+; CHECK-DAG: %[[#Scope_Workgroup:]] = OpConstant %[[#Int]] 2{{$}}
+; CHECK-DAG: %[[#Scope_Subgroup:]] = OpConstant %[[#Int]] 3{{$}}
+; CHECK-DAG: %[[#Scope_Invocation:]] = OpConstant %[[#Int]] 4{{$}}
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528{{$}}
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
new file mode 100644
index 0000000000000..f6fd4cf3ed6ab
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
@@ -0,0 +1,67 @@
+; A module may contain uinc_wrap/udec_wrap atomics with mutually incompatible
+; signatures, while SPIR-V resolves an imported function by its linkage name
+; alone. Verify that the _p<addrspace>_i<width> suffix keeps them apart: each
+; distinct (address space, value type) combination gets its own declaration,
+; and every call site of a given combination shares that one declaration.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
+; CHECK-DAG: %[[#Value64:]] = OpConstant %[[#Long]] 42
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; The storage class contributes to the memory semantics, so an atomic on a
+; Workgroup pointer carries WorkgroupMemory (256) where a CrossWorkgroup one
+; carries CrossWorkgroupMemory (512).
+; CHECK-DAG: %[[#MemSem_Relaxed_Local:]] = OpConstant %[[#Int]] 256
+; CHECK-DAG: %[[#MemSem_Relaxed:]] = OpConstant %[[#Int]] 512
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UIncFnLocal:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p3_i32" Import
+; CHECK-DAG: OpDecorate %[[#UIncFn64:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i64" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+ at lui = common dso_local addrspace(3) global i32 0, align 4
+ at ul = common dso_local addrspace(1) global i64 0, align 8
+
+; Two atomics of the same value type in different address spaces need two
+; incompatible signatures, so they must resolve to two distinct declarations.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFnLocal]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed_Local]] %[[#Value]]
+define dso_local spir_func void @mixed_addrspace() local_unnamed_addr {
+entry:
+ %g = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %l = atomicrmw uinc_wrap ptr addrspace(3) @lui, i32 42 monotonic
+ ret void
+}
+
+; Likewise for two atomics in the same address space with different value
+; widths: the i64 one must not reuse the i32 declaration, and its call must
+; yield an i64 result.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Long]] %[[#UIncFn64]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value64]]
+define dso_local spir_func void @mixed_width() local_unnamed_addr {
+entry:
+ %a = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %b = atomicrmw uinc_wrap ptr addrspace(1) @ul, i64 42 monotonic
+ ret void
+}
+
+; uinc_wrap and udec_wrap on the same (address space, value type) are still two
+; separate symbols, and repeated call sites reuse the single declaration.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+define dso_local spir_func void @shared_declaration() local_unnamed_addr {
+entry:
+ %a = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %b = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
index ed53ea525e39e..66c01ef739a60 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -1,56 +1,41 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; Verify that on AMD targets atomicrmw uinc_wrap/udec_wrap lower to
+; OpFunctionCall to
+; __translate_spirv_atomic_uinc_wrap_*/__translate_spirv_atomic_udec_wrap_* with
+; Import linkage, rather than being expanded to a CmpXChg loop. The name carries
+; a _p<addrspace>_i<width> suffix, because a module may need several mutually
+; incompatible signatures while SPIR-V resolves an imported function by its
+; linkage name alone.
+;
+; The helper is an AMD extension, so non-AMD targets keep the generic expansion
+; instead; that is covered by atomicrmw-uinc-udec-wrap-non-amd.ll.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Bool:]] = OpTypeBool
; CHECK-DAG: %[[#PointerType:]] = OpTypePointer CrossWorkgroup %[[#Int]]
; CHECK-DAG: %[[#MemSem_SequentiallyConsistent:]] = OpConstant %[[#Int]] 528
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
-; CHECK-DAG: %[[#One:]] = OpConstant %[[#Int]] 1
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#PointerType]] CrossWorkgroup
; CHECK-DAG: %[[#AllOnes:]] = OpConstant %[[#Int]] 4294967295
- at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK-DAG: OpDecorate %[[#UIncWrapFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecWrapFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Add:]] = OpIAdd %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#GE:]] = OpUGreaterThanEqual %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#GE]] %[[#Scope_CrossDevice]] %[[#Add]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SequentiallyConsistent]] %[[#Value]]
define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
ret void
}
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Sub:]] = OpISub %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#Equal:]] = OpIEqual %[[#Bool]] %[[#Phi]] %[[#Scope_CrossDevice]]
-; CHECK: %[[#GT:]] = OpUGreaterThan %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Or:]] = OpLogicalOr %[[#Bool]] %[[#Equal]] %[[#GT]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#Or]] %[[#Value]] %[[#Sub]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
-
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SequentiallyConsistent]] %[[#Value]]
define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
>From 915572e280751cfb18c363b644c8baa76825ad08 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 04:34:10 -0500
Subject: [PATCH 2/4] [SPIRV] Preserve AMDGPU atomic metadata through
NonSemantic.AuxData
AMDGPU atomic metadata (amdgpu.no.fine.grained.memory,
amdgpu.no.remote.memory, amdgpu.ignore.denormal.mode) was encoded as
UserSemantic decorations on the atomic result. UserSemantic is a semantic
decoration that consumers are required to understand, so emitting it for
data that is purely advisory is a misuse, and it was emitted unconditionally
rather than under the existing -spirv-preserve-auxdata flag.
Encode the metadata as NonSemantic.AuxData instead, adding a new
InstructionMetadata opcode (5) whose operands are the target instruction's
result <id> and an OpString naming the metadata. SPIRVEmitIntrinsics tags
the atomic with an internal AuxDataInstructionMetadata sentinel decoration,
SPIRVModuleAnalysis intercepts the sentinel and records the target register,
and SPIRVAuxDataHandler emits the OpString and OpExtInst. The sentinel is
never emitted in SPIR-V output. Emission is gated on -spirv-preserve-auxdata,
so the default output is unchanged.
---
llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp | 7 +--
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp | 32 ++++++++++-
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h | 6 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 22 +++----
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 21 +++++++
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h | 25 ++++++++
.../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 2 +
.../amdgcnspirv-atomic-metadata-decoration.ll | 21 ++++---
...preserve-auxdata-amdgpu-atomic-metadata.ll | 57 +++++++++++++++++++
...-auxdata-atomic-metadata-generic-target.ll | 36 ++++++++++++
10 files changed, 198 insertions(+), 31 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 040a6858b9009..3f7bf566f42b5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -886,11 +886,8 @@ void SPIRVAsmPrinter::outputModuleSections() {
MAI = &getAnalysis<SPIRVModuleAnalysis>().MAI;
assert(ST && TII && MAI && M && "Module analysis is required");
- if (!AuxDataHandler) {
- auto Handler = std::make_unique<SPIRVAuxDataHandler>(*this, *M);
- if (Handler->hasWork())
- AuxDataHandler = std::move(Handler);
- }
+ if (!AuxDataHandler && spirvPreserveAuxData())
+ AuxDataHandler = std::make_unique<SPIRVAuxDataHandler>(*this, *M);
// Let the NSDI handler add its extension and ext inst import entry to MAI
// before the module header sections are emitted.
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
index 469c261ac433e..b7af23862d7d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
@@ -63,11 +63,11 @@ SPIRVAuxDataHandler::SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M)
LinkagePreservedGOs.push_back(&GO);
}
-bool SPIRVAuxDataHandler::hasWork() const { return SPVPreserveAuxData; }
+bool llvm::spirvPreserveAuxData() { return SPVPreserveAuxData; }
void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI) {
- if (!hasWork())
+ if (!spirvPreserveAuxData())
return;
if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_non_semantic_info)) {
if (SPVPreserveAuxData)
@@ -76,7 +76,7 @@ void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
return;
}
MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
- if (!MAI.ExtInstSetMap.count(NonSemanticAuxDataSet))
+ if (!MAI.ExtInstSetMap.contains(NonSemanticAuxDataSet))
MAI.ExtInstSetMap[NonSemanticAuxDataSet] = MAI.getNextIDRegister();
}
@@ -181,6 +181,22 @@ void SPIRVAuxDataHandler::emitAuxDataStrings(SPIRV::ModuleAnalysisInfo &MAI) {
collectAttributesFor(&GO, MAI);
collectMetadataFor(&GO, MDNames, MAI);
}
+ // Only a handful of distinct metadata names exist, one per AMDGPUAtomicMDKind
+ // enumerator. Track which we've seen so we can stop once every name has been
+ // emitted, instead of scanning potentially thousands of records with
+ // redundant hash lookups.
+ constexpr unsigned AllMDKindsSeen =
+ (1u << (static_cast<unsigned>(
+ SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind::Last) +
+ 1)) -
+ 1;
+ unsigned SeenMask = 0;
+ for (const auto &Rec : MAI.InstrAuxDataRecords) {
+ SeenMask |= 1u << static_cast<unsigned>(Rec.Kind);
+ getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ if (SeenMask == AllMDKindsSeen)
+ break;
+ }
}
void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
@@ -201,6 +217,16 @@ void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
emitAuxDataExtInst(Rec.Opcode, VoidTypeReg, ExtSetReg, Operands, MAI);
}
+ for (const auto &Rec : MAI.InstrAuxDataRecords) {
+ MCRegister TargetReg = MAI.getRegisterAlias(Rec.MF, Rec.TargetReg);
+ if (!TargetReg.isValid())
+ continue;
+ MCRegister MDNameReg =
+ getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ emitAuxDataExtInst(InstructionMetadataOpcode, VoidTypeReg, ExtSetReg,
+ {TargetReg, MDNameReg}, MAI);
+ }
+
if (LinkagePreservedGOs.empty())
return;
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
index 1d6ba998e6c2e..6ad5c45f6a086 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
@@ -29,7 +29,6 @@ namespace llvm {
class AsmPrinter;
class Constant;
-class Function;
class GlobalObject;
class Module;
class SPIRVSubtarget;
@@ -42,14 +41,13 @@ enum AuxDataOpcode : int64_t {
GlobalVariableMetadataOpcode = 2,
GlobalVariableAttributeOpcode = 3,
LinkageOpcode = 4,
+ InstructionMetadataOpcode = 5,
};
class SPIRVAuxDataHandler {
public:
SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M);
- bool hasWork() const;
-
/// Register extension + ext-inst-set; call before output of section 1.
void prepareModuleOutput(const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI);
@@ -111,6 +109,8 @@ class SPIRVAuxDataHandler {
SPIRV::ModuleAnalysisInfo &MAI);
};
+bool spirvPreserveAuxData();
+
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRVAUXDATAHANDLER_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index a0449e7a6f5f6..34544783faac6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -13,6 +13,7 @@
#include "SPIRVEmitIntrinsics.h"
#include "SPIRV.h"
+#include "SPIRVAuxDataHandler.h"
#include "SPIRVBuiltins.h"
#include "SPIRVSubtarget.h"
#include "SPIRVTargetMachine.h"
@@ -3071,27 +3072,26 @@ void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
{I->getType()},
{I, MetadataAsValue::get(I->getContext(), MD)});
}
- if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
- isa<AtomicRMWInst>(I)) {
- // If present, we encode AMDGPU atomic metadata as UserSemantic string
- // decorations, which will be parsed during reverse translation.
- auto &Ctx = B.getContext();
- auto *US = ConstantAsMetadata::get(
- ConstantInt::get(B.getInt32Ty(), SPIRV::Decoration::UserSemantic));
+ if (spirvPreserveAuxData() && isa<AtomicRMWInst>(I)) {
+ LLVMContext &Ctx = B.getContext();
+ auto *AuxMD = ConstantAsMetadata::get(ConstantInt::get(
+ B.getInt32Ty(), SPIRV::Decoration::AuxDataInstructionMetadata));
SmallVector<Metadata *> MDs;
if (I->hasMetadata("amdgpu.no.fine.grained.memory"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.no.fine.grained.memory")}));
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.no.fine.grained.memory")}));
if (I->hasMetadata("amdgpu.no.remote.memory"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.no.remote.memory")}));
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.no.remote.memory")}));
if (I->hasMetadata("amdgpu.ignore.denormal.mode"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.ignore.denormal.mode")}));
- if (!MDs.empty())
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.ignore.denormal.mode")}));
+ if (!MDs.empty()) {
+ setInsertPointAfterDef(B, I);
B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()},
{I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDs))});
+ }
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 2836a2cc43ef0..7cee61f9e0a55 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -26,6 +26,7 @@
#include "SPIRVTargetMachine.h"
#include "SPIRVUtils.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -747,6 +748,26 @@ void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
} else if (TII->isAliasingInstr(MI)) {
collectOtherInstr(MI, MAI, SPIRV::MB_AliasingInsts, IS);
} else if (TII->isDecorationInstr(MI)) {
+ if (MI.getOpcode() == SPIRV::OpDecorate &&
+ MI.getOperand(1).getImm() ==
+ static_cast<unsigned>(
+ SPIRV::Decoration::AuxDataInstructionMetadata)) {
+ MAI.setSkipEmission(&MI);
+ std::string Str = getStringImm(MI, 2);
+ using AMDMD = SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind;
+ auto MaybeKind =
+ StringSwitch<std::optional<AMDMD>>(Str)
+ .Case("amdgpu.no.fine.grained.memory",
+ AMDMD::NoFineGrainedMemory)
+ .Case("amdgpu.no.remote.memory", AMDMD::NoRemoteMemory)
+ .Case("amdgpu.ignore.denormal.mode",
+ AMDMD::IgnoreDenormalMode)
+ .Default(std::nullopt);
+ if (MaybeKind)
+ MAI.InstrAuxDataRecords.push_back(
+ {MF, MI.getOperand(0).getReg(), *MaybeKind});
+ continue;
+ }
collectOtherInstr(MI, MAI, SPIRV::MB_Annotations, IS);
collectFuncNames(MI, &F);
} else if (TII->isConstantInstr(MI)) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index 6559b5cfc7457..8114aa63610d1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -167,6 +167,20 @@ struct ModuleAnalysisInfo {
DenseMap<const Function *, SPIRV::FPFastMathDefaultInfoVector>
FPFastMathDefaultInfoMap;
+ enum class AMDGPUAtomicMDKind : uint8_t {
+ NoFineGrainedMemory,
+ NoRemoteMemory,
+ IgnoreDenormalMode,
+
+ Last = IgnoreDenormalMode,
+ };
+ struct InstrAuxDataRecord {
+ const MachineFunction *MF;
+ Register TargetReg;
+ AMDGPUAtomicMDKind Kind;
+ };
+ SmallVector<InstrAuxDataRecord> InstrAuxDataRecords;
+
MCRegister getGlobalObjReg(const GlobalObject *GO) {
assert(GO && "GlobalObject is null");
return GlobalObjMap.lookup(GO);
@@ -211,6 +225,17 @@ struct ModuleAnalysisInfo {
It->second = getNextIDRegister();
return It->second;
}
+ static StringRef getAMDGPUAtomicMDName(AMDGPUAtomicMDKind Kind) {
+ switch (Kind) {
+ case AMDGPUAtomicMDKind::NoFineGrainedMemory:
+ return "amdgpu.no.fine.grained.memory";
+ case AMDGPUAtomicMDKind::NoRemoteMemory:
+ return "amdgpu.no.remote.memory";
+ case AMDGPUAtomicMDKind::IgnoreDenormalMode:
+ return "amdgpu.ignore.denormal.mode";
+ }
+ llvm_unreachable("unknown AMDGPUAtomicMDKind");
+ }
};
} // namespace SPIRV
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 7da57bc1b2d47..cf4393c4bb884 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -1435,6 +1435,8 @@ defm FunctionFloatingPointModeINTEL : DecorationOperand<6080, 0, 0, [], [Functio
defm AliasScopeINTEL : DecorationOperand<5914, 0, 0, [], [MemoryAccessAliasingINTEL]>;
defm NoAliasINTEL : DecorationOperand<5915, 0, 0, [], [MemoryAccessAliasingINTEL]>;
defm FPMaxErrorDecorationINTEL : DecorationOperand<6170, 0, 0, [], [FPMaxErrorINTEL]>;
+// Internal-only sentinel; intercepted by SPIRVModuleAnalysis, never emitted.
+defm AuxDataInstructionMetadata : DecorationOperand<0xFFFF, 0, 0, [], []>;
//===----------------------------------------------------------------------===//
// Multiclass used to define BuiltIn enum values and at the same time
diff --git a/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll b/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
index 9c6300f374261..3d679fe7e2281 100644
--- a/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
+++ b/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
@@ -1,18 +1,21 @@
-; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; Without -spirv-preserve-auxdata, AMDGPU atomic metadata must not appear
+; as UserSemantic decorations or any other form in the SPIR-V output.
-; CHECK: OpDecorate %[[#Add:]] UserSemantic "amdgpu.no.fine.grained.memory"
-; CHECK-NEXT: OpDecorate %[[#Add]] UserSemantic "amdgpu.no.remote.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd:]] UserSemantic "amdgpu.no.fine.grained.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd]] UserSemantic "amdgpu.no.remote.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd]] UserSemantic "amdgpu.ignore.denormal.mode"
-; CHECK: %[[#Add]] = OpAtomicIAdd
-; CHECK: %[[#FAdd]] = OpAtomicFAddEXT
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-NOT: amdgpu.no.fine.grained.memory
+; CHECK-NOT: amdgpu.no.remote.memory
+; CHECK-NOT: amdgpu.ignore.denormal.mode
+; CHECK: %[[#Add:]] = OpAtomicIAdd
+; CHECK: %[[#FAdd:]] = OpAtomicFAddEXT
+; CHECK: %[[#Xchg:]] = OpAtomicExchange
define spir_func void @foo(ptr addrspace(1) %p) {
entry:
%atomic.add = atomicrmw add ptr addrspace(1) %p, i32 1 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
%atomic.fadd = atomicrmw fadd ptr addrspace(1) %p, float 1.0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+ %atomic.xchg = atomicrmw xchg ptr addrspace(1) %p, i32 1 seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
new file mode 100644
index 0000000000000..5bd2211db417d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
@@ -0,0 +1,57 @@
+; Test that AMDGPU atomic metadata is preserved as NonSemantic.AuxData
+; InstructionMetadata (opcode 5).
+
+; Positive: with -spirv-preserve-auxdata, metadata emitted as AuxData.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; Negative: without -spirv-preserve-auxdata, no metadata strings.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+; OFF-NOT: amdgpu.no.remote.memory
+; OFF-NOT: amdgpu.ignore.denormal.mode
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
+; CHECK-DAG: %[[#md_idn:]] = OpString "amdgpu.ignore.denormal.mode"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+
+; Integer atomic (add) with two metadata kinds.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res]] %[[#md_nrm]]
+
+; Float atomic (fadd) with all three metadata kinds.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_nrm]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_idn]]
+
+; Atomic (xchg) with only one metadata kind.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#xchg_res:]] %[[#md_nfg]]
+
+; The atomic instructions themselves (forward-referenced by AuxData above).
+; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
+; CHECK-DAG: %[[#fadd_res]] = OpAtomicFAddEXT
+; CHECK-DAG: %[[#xchg_res]] = OpAtomicExchange
+
+
+define amdgpu_kernel void @test_iadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw add ptr addrspace(1) %ptr, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret void
+}
+
+define amdgpu_kernel void @test_fadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw fadd ptr addrspace(1) %ptr, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+ ret void
+}
+
+define amdgpu_kernel void @test_xchg(ptr addrspace(1) %ptr) {
+ %val = atomicrmw xchg ptr addrspace(1) %ptr, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
new file mode 100644
index 0000000000000..9966efaf3482f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
@@ -0,0 +1,36 @@
+; NonSemantic.AuxData is non-semantic by construction: a consumer that does not
+; understand the extended instruction set ignores it. Preserving instruction
+; metadata is therefore not restricted to AMD targets, even though the metadata
+; names that currently benefit are amdgpu.*. Verify that -spirv-preserve-auxdata
+; emits it on a generic target too.
+;
+; The AMD-target behaviour and the full set of metadata kinds are covered by
+; preserve-auxdata-amdgpu-atomic-metadata.ll.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; Without the option nothing is emitted, on any target.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
+
+; No spirv-val run with the option on: the AuxData instruction forward-references
+; the atomic's result <id>, which spirv-val rejects. See the CHECK-INVALID pin in
+; preserve-auxdata-amdgpu-atomic-metadata.ll.
+
+define spir_func void @test_iadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw add ptr addrspace(1) %ptr, i32 1 monotonic, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
>From 5634c0c926dd3be8cd56901a16dc5c788ef6f6a2 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 07:32:02 -0500
Subject: [PATCH 3/4] [SPIRV] Validate the AuxData atomic-metadata test with
spirv-val
The test never ran spirv-val, so the module-scope forward reference from
InstructionMetadata to an in-function atomic result went unnoticed. Add a
spirv-val run for the default output, which validates, and pin the
preserve-auxdata output's current rejection with "not spirv-val" plus a comment
explaining the forward reference, so the behaviour change is visible whichever
way it is resolved.
---
...preserve-auxdata-amdgpu-atomic-metadata.ll | 21 +++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
index 5bd2211db417d..5d76ff1965e1b 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
@@ -15,6 +15,27 @@
; OFF-NOT: amdgpu.no.remote.memory
; OFF-NOT: amdgpu.ignore.denormal.mode
+; Default output, with the feature off, validates.
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - -filetype=obj \
+; RUN: | spirv-val %}
+
+; The AuxData instructions go in the module-level section, but the Target
+; operand of InstructionMetadata is the result <id> of an atomic defined inside
+; a function body, so it is a forward reference. That is intentional -- the
+; instruction is non-semantic and its result is never consumed, and
+; NonSemantic.AuxData.asciidoc permits it -- but spirv-val implements no such
+; relaxation and rejects the module. Pin that down so the day validation starts
+; passing is not silent: drop the "not" and the CHECK-INVALID prefix once
+; SPIRV-Tools accepts the forward reference, or once the instructions are moved
+; into the function body after their target.
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - -filetype=obj | not spirv-val 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INVALID %}
+
+; CHECK-INVALID: has not been defined
+
; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
>From a940e98794fc0b146841eeca4fd9e02ba96ea1a9 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 07:35:07 -0500
Subject: [PATCH 4/4] [SPIRV] Test AuxData metadata on the uinc_wrap/udec_wrap
lowering
uinc_wrap/udec_wrap do not reach the generic atomicrmw path -- they become an
OpFunctionCall to an imported helper -- so the AuxData records must attach to
the call's result <id> instead. Nothing covered that combination; each half was
tested on its own. Add the joint case, on top of the lowering it depends on.
---
...reserve-auxdata-uinc-udec-wrap-metadata.ll | 67 +++++++++++++++++++
1 file changed, 67 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
new file mode 100644
index 0000000000000..1f436f53bea9d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
@@ -0,0 +1,67 @@
+; Test that atomicrmw uinc_wrap/udec_wrap with AMDGPU metadata emit both
+; OpFunctionCall (for the atomic) and AuxData InstructionMetadata (for the
+; metadata), all gated on -spirv-preserve-auxdata.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+; OFF-NOT: amdgpu.no.remote.memory
+
+; Default output, with the feature off, validates.
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 \
+; RUN: -mtriple=spirv64-amd-amdhsa --spirv-ext=+SPV_KHR_non_semantic_info \
+; RUN: %s -o - -filetype=obj | spirv-val %}
+
+; As in preserve-auxdata-amdgpu-atomic-metadata.ll, the AuxData instructions
+; sit in the module-level section and forward-reference a result <id> defined
+; inside a function body -- here the OpFunctionCall standing in for the
+; atomicrmw. spirv-val does not accept that, so pin the rejection rather than
+; leave the module unvalidated. Drop the "not" and the CHECK-INVALID prefix
+; when the forward reference is resolved.
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 \
+; RUN: -mtriple=spirv64-amd-amdhsa --spirv-ext=+SPV_KHR_non_semantic_info \
+; RUN: -spirv-preserve-auxdata %s -o - -filetype=obj | not spirv-val 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INVALID %}
+
+; CHECK-INVALID: has not been defined
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+; AuxData for the uinc_wrap result.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res]] %[[#md_nrm]]
+
+; AuxData for the udec_wrap result.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#udec_res:]] %[[#md_nfg]]
+
+; The function calls themselves.
+; CHECK-DAG: %[[#uinc_res]] = OpFunctionCall %[[#]] %[[#UIncFn]]
+; CHECK-DAG: %[[#udec_res]] = OpFunctionCall %[[#]] %[[#UDecFn]]
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define amdgpu_kernel void @test_uinc_wrap() {
+entry:
+ %uinc = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret void
+}
+
+define amdgpu_kernel void @test_udec_wrap() {
+entry:
+ %udec = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
More information about the llvm-commits
mailing list