[llvm] [SPIRV] Preserve AMDGPU metadata through NonSemantic.AuxData (PR #215510)
Marcos Maronas via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 08:50:54 PDT 2026
https://github.com/maarquitos14 updated https://github.com/llvm/llvm-project/pull/215510
>From b43c9b845b13dd8181d826c2ae09fb9e96cb774b Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 04:33:00 -0500
Subject: [PATCH 01/10] [SPIRV] Translate atomicrmw {uinc,udec}_wrap as
OpFunctionCall
atomicrmw uinc_wrap/udec_wrap had no direct SPIR-V representation and were
expanded into a compare-exchange loop before reaching the SPIRV backend,
which prevents targets with native wrapping increment/decrement atomics
(e.g. AMDGPU ds_inc_u32/ds_dec_u32) from recovering the original operation.
Keep the atomicrmw intact by returning AtomicExpansionKind::None from
shouldExpandAtomicRMWInIR, and translate it in SPIRVEmitIntrinsics to a
SPIR_FUNC call to __translate_spirv_atomic_uinc_wrap /
__translate_spirv_atomic_udec_wrap, with scope and memory semantics derived
from the instruction's syncscope and ordering. The imported declaration is
emitted with LinkageAttributes Import. All other atomicrmw operations are
unaffected.
The helpers deliberately avoid the __spirv_ prefix: that namespace is
reserved for SPIR-V friendly IR, where a name maps to a SPIR-V opcode (see
__spirv_AtomicUMin in SPIRVBuiltins.td). No such opcode exists for these
operations, so the __translate_ prefix already used for similar
translation-only symbols is used instead.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 56 ++++++++++++++++
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 5 ++
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 2 +
llvm/lib/Target/SPIRV/SPIRVUtils.h | 3 +
.../SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll | 30 +++++++++
.../atomicrmw-uinc-udec-wrap-orderings.ll | 66 ++++++++++++++++++
.../SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll | 59 ++++++++++++++++
.../atomicrmw-uinc-udec-wrap-signatures.ll | 67 +++++++++++++++++++
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 55 ++++++---------
9 files changed, 308 insertions(+), 35 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 0f097c742fac2..7fcc6a9950a18 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -394,6 +394,7 @@ class SPIRVEmitIntrinsicsImpl
Instruction *visitStoreInst(StoreInst &I);
Instruction *visitAllocaInst(AllocaInst &I);
Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
Instruction *visitUnreachableInst(UnreachableInst &I);
Instruction *visitCallInst(CallInst &I);
@@ -2627,6 +2628,61 @@ SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
return NewI;
}
+Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
+ auto Op = I.getOperation();
+ if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
+ return &I;
+
+ // Carrying these across the SPIR-V boundary as a call to an imported helper
+ // is an AMD extension: there is no SPIR-V opcode for them, so a consumer has
+ // to recognize the helper by name to make sense of the module. Restrict it to
+ // AMD targets and let everyone else keep the generic expansion.
+ if (!isAMDTarget(TM.getTargetTriple()))
+ return &I;
+
+ Module *M = I.getModule();
+ IRBuilder<> B(I.getParent());
+ B.SetInsertPoint(&I);
+
+ const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(*I.getFunction());
+ unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
+
+ uint32_t Scope = static_cast<uint32_t>(
+ getMemScope(TM.getTargetTriple(), I.getContext(), I.getSyncScopeID()));
+ uint32_t ScSem = static_cast<uint32_t>(
+ getMemSemanticsForStorageClass(addressSpaceToStorageClass(AS, ST)));
+ uint32_t MemSem =
+ static_cast<uint32_t>(getMemSemantics(I.getOrdering())) | ScSem;
+
+ std::string FuncName = (Op == AtomicRMWInst::UIncWrap)
+ ? "__translate_spirv_atomic_uinc_wrap"
+ : "__translate_spirv_atomic_udec_wrap";
+
+ Type *ValTy = I.getValOperand()->getType();
+ Type *PtrTy = I.getPointerOperand()->getType();
+ // Encode the address space and the value type in the name, the same way
+ // lowerLLVMIntrinsicName() does for spirv.llvm_memset_p1_i64. A module may
+ // need several mutually incompatible signatures, while SPIR-V resolves an
+ // imported function by its linkage name alone.
+ FuncName += "_p" + std::to_string(AS) + "_i" +
+ std::to_string(ValTy->getIntegerBitWidth());
+
+ Type *Int32Ty = B.getInt32Ty();
+ SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
+ FunctionType *FT = FunctionType::get(ValTy, ArgTys, false);
+ FunctionCallee FC = M->getOrInsertFunction(FuncName, FT);
+ if (auto *F = dyn_cast<Function>(FC.getCallee()))
+ F->setCallingConv(CallingConv::SPIR_FUNC);
+
+ SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt32(Scope),
+ B.getInt32(MemSem), I.getValOperand()};
+ CallInst *CI = B.CreateCall(FC, Args);
+ CI->setCallingConv(CallingConv::SPIR_FUNC);
+
+ replaceAllUsesWithAndErase(B, &I, CI);
+ return CI;
+}
+
static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
auto *CI = dyn_cast<CallInst>(&I);
if (!CI)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 79be9dee82546..e5e1169811bbb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -688,6 +688,11 @@ SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
return AtomicExpansionKind::None;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
+ // On AMD targets these are translated into a call to an imported helper by
+ // SPIRVEmitIntrinsics, so they must survive to that point unexpanded. Any
+ // other target has no such helper and needs the generic expansion.
+ return isAMDTarget(STI.getTargetTriple()) ? AtomicExpansionKind::None
+ : AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Nand:
return AtomicExpansionKind::CmpXChg;
default:
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 75fb210a19244..c917f48711bad 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -449,6 +449,8 @@ SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord) {
llvm_unreachable(nullptr);
}
+bool isAMDTarget(const Triple &T) { return T.getVendor() == Triple::AMD; }
+
uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem,
uint32_t StorageClassSem) {
bool DropStorageClass =
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index abae9cc3da23a..374771cda11f4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -292,6 +292,9 @@ uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem,
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx,
SyncScope::ID Id);
+// Returns true if T targets an AMD SPIR-V flavour.
+bool isAMDTarget(const Triple &T);
+
// Find def instruction for the given ConstReg, walking through
// spv_track_constant and ASSIGN_TYPE instructions. Updates ConstReg by def
// of OpConstant instruction.
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
new file mode 100644
index 0000000000000..fde6dd15cc0d5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-non-amd.ll
@@ -0,0 +1,30 @@
+; Translating atomicrmw uinc_wrap/udec_wrap into a call to an imported helper is
+; an AMD extension: there is no SPIR-V opcode for these, so a consumer has to
+; recognize the helper by name to make sense of the module. Verify that a
+; non-AMD target does not emit it, and instead falls back to the generic CmpXChg
+; expansion. The AMD behaviour is covered by atomicrmw-uinc-udec-wrap.ll.
+;
+; --implicit-check-not applies over the whole module, unlike a CHECK-NOT, which
+; would only cover the input up to the first positive match below.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --implicit-check-not=__translate_spirv_atomic
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --implicit-check-not=__translate_spirv_atomic
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+; Both operations expand to an OpAtomicCompareExchange retry loop.
+; CHECK: OpAtomicCompareExchange
+define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
+entry:
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+ ret void
+}
+
+; CHECK: OpAtomicCompareExchange
+define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
+entry:
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
new file mode 100644
index 0000000000000..80c771b31eb54
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
@@ -0,0 +1,66 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode memory
+; orderings in the function call arguments.
+; CrossWorkgroupMemory = 0x200 = 512
+; Monotonic (Relaxed) = 0x000 -> with CrossWorkgroup: 512
+; Acquire = 0x002 -> with CrossWorkgroup: 514
+; Release = 0x004 -> with CrossWorkgroup: 516
+; AcquireRelease = 0x008 -> with CrossWorkgroup: 520
+; SequentiallyConsistent = 0x010 -> with CrossWorkgroup: 528
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#MemSem_Monotonic:]] = OpConstant %[[#Int]] 512
+; CHECK-DAG: %[[#MemSem_Acquire:]] = OpConstant %[[#Int]] 514
+; CHECK-DAG: %[[#MemSem_Release:]] = OpConstant %[[#Int]] 516
+; CHECK-DAG: %[[#MemSem_AcqRel:]] = OpConstant %[[#Int]] 520
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
new file mode 100644
index 0000000000000..c748e47807f63
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
@@ -0,0 +1,59 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode scopes.
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#Scope_Device:]] = OpConstant %[[#Int]] 1{{$}}
+; CHECK-DAG: %[[#Scope_Workgroup:]] = OpConstant %[[#Int]] 2{{$}}
+; CHECK-DAG: %[[#Scope_Subgroup:]] = OpConstant %[[#Int]] 3{{$}}
+; CHECK-DAG: %[[#Scope_Invocation:]] = OpConstant %[[#Int]] 4{{$}}
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528{{$}}
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
new file mode 100644
index 0000000000000..f6fd4cf3ed6ab
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
@@ -0,0 +1,67 @@
+; A module may contain uinc_wrap/udec_wrap atomics with mutually incompatible
+; signatures, while SPIR-V resolves an imported function by its linkage name
+; alone. Verify that the _p<addrspace>_i<width> suffix keeps them apart: each
+; distinct (address space, value type) combination gets its own declaration,
+; and every call site of a given combination shares that one declaration.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
+; CHECK-DAG: %[[#Value64:]] = OpConstant %[[#Long]] 42
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; The storage class contributes to the memory semantics, so an atomic on a
+; Workgroup pointer carries WorkgroupMemory (256) where a CrossWorkgroup one
+; carries CrossWorkgroupMemory (512).
+; CHECK-DAG: %[[#MemSem_Relaxed_Local:]] = OpConstant %[[#Int]] 256
+; CHECK-DAG: %[[#MemSem_Relaxed:]] = OpConstant %[[#Int]] 512
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UIncFnLocal:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p3_i32" Import
+; CHECK-DAG: OpDecorate %[[#UIncFn64:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i64" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+ at lui = common dso_local addrspace(3) global i32 0, align 4
+ at ul = common dso_local addrspace(1) global i64 0, align 8
+
+; Two atomics of the same value type in different address spaces need two
+; incompatible signatures, so they must resolve to two distinct declarations.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFnLocal]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed_Local]] %[[#Value]]
+define dso_local spir_func void @mixed_addrspace() local_unnamed_addr {
+entry:
+ %g = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %l = atomicrmw uinc_wrap ptr addrspace(3) @lui, i32 42 monotonic
+ ret void
+}
+
+; Likewise for two atomics in the same address space with different value
+; widths: the i64 one must not reuse the i32 declaration, and its call must
+; yield an i64 result.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Long]] %[[#UIncFn64]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value64]]
+define dso_local spir_func void @mixed_width() local_unnamed_addr {
+entry:
+ %a = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %b = atomicrmw uinc_wrap ptr addrspace(1) @ul, i64 42 monotonic
+ ret void
+}
+
+; uinc_wrap and udec_wrap on the same (address space, value type) are still two
+; separate symbols, and repeated call sites reuse the single declaration.
+
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#Value]]
+define dso_local spir_func void @shared_declaration() local_unnamed_addr {
+entry:
+ %a = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ %b = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
index ed53ea525e39e..66c01ef739a60 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -1,56 +1,41 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; Verify that on AMD targets atomicrmw uinc_wrap/udec_wrap lower to
+; OpFunctionCall to
+; __translate_spirv_atomic_uinc_wrap_*/__translate_spirv_atomic_udec_wrap_* with
+; Import linkage, rather than being expanded to a CmpXChg loop. The name carries
+; a _p<addrspace>_i<width> suffix, because a module may need several mutually
+; incompatible signatures while SPIR-V resolves an imported function by its
+; linkage name alone.
+;
+; The helper is an AMD extension, so non-AMD targets keep the generic expansion
+; instead; that is covered by atomicrmw-uinc-udec-wrap-non-amd.ll.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Bool:]] = OpTypeBool
; CHECK-DAG: %[[#PointerType:]] = OpTypePointer CrossWorkgroup %[[#Int]]
; CHECK-DAG: %[[#MemSem_SequentiallyConsistent:]] = OpConstant %[[#Int]] 528
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
-; CHECK-DAG: %[[#One:]] = OpConstant %[[#Int]] 1
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#PointerType]] CrossWorkgroup
; CHECK-DAG: %[[#AllOnes:]] = OpConstant %[[#Int]] 4294967295
- at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK-DAG: OpDecorate %[[#UIncWrapFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecWrapFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Add:]] = OpIAdd %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#GE:]] = OpUGreaterThanEqual %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#GE]] %[[#Scope_CrossDevice]] %[[#Add]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SequentiallyConsistent]] %[[#Value]]
define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
ret void
}
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Sub:]] = OpISub %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#Equal:]] = OpIEqual %[[#Bool]] %[[#Phi]] %[[#Scope_CrossDevice]]
-; CHECK: %[[#GT:]] = OpUGreaterThan %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Or:]] = OpLogicalOr %[[#Bool]] %[[#Equal]] %[[#GT]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#Or]] %[[#Value]] %[[#Sub]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
-
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SequentiallyConsistent]] %[[#Value]]
define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
>From 885561d48632255531a8a7de976f91be9e96a848 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Tue, 11 Aug 2026 10:27:21 -0500
Subject: [PATCH 02/10] Add support for vector types and corresponding tests.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 12 +++++++---
.../atomicrmw-uinc-udec-wrap-oversized.ll | 22 +++++++++++++++++++
.../atomicrmw-uinc-udec-wrap-signatures.ll | 21 ++++++++++++++++--
3 files changed, 50 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 7fcc6a9950a18..dc2e8cb30e986 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2663,9 +2663,15 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
// Encode the address space and the value type in the name, the same way
// lowerLLVMIntrinsicName() does for spirv.llvm_memset_p1_i64. A module may
// need several mutually incompatible signatures, while SPIR-V resolves an
- // imported function by its linkage name alone.
- FuncName += "_p" + std::to_string(AS) + "_i" +
- std::to_string(ValTy->getIntegerBitWidth());
+ // imported function by its linkage name alone. The value may also be a fixed
+ // vector of integers, spelled the LLVM way: _p1_v2i32. Anything wider than
+ // the target's maximum atomic size never reaches here, because AtomicExpand
+ // rejects it first.
+ std::string TypeSuffix;
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ValTy))
+ TypeSuffix = "v" + std::to_string(VecTy->getNumElements());
+ TypeSuffix += "i" + std::to_string(ValTy->getScalarSizeInBits());
+ FuncName += "_p" + std::to_string(AS) + "_" + TypeSuffix;
Type *Int32Ty = B.getInt32Ty();
SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll
new file mode 100644
index 0000000000000..d258bd7d829ba
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-oversized.ll
@@ -0,0 +1,22 @@
+; The SPIR-V target supports no atomic wider than 64 bits
+; (SPIRVTargetLowering sets setMaxAtomicSizeInBitsSupported(64)), so a vector
+; operand that does not fit is rejected by AtomicExpandPass before the
+; uinc_wrap/udec_wrap helper lowering ever sees it. Verify that carrying these
+; two operations across as an imported helper on AMD targets does not smuggle
+; an over-limit atomic past that check: the diagnostic is the same one any
+; other target and any other atomicrmw operation gets. The supported vector
+; widths are covered by atomicrmw-uinc-udec-wrap-signatures.ll.
+
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - 2>&1 | FileCheck %s
+; RUN: not llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - 2>&1 | FileCheck %s
+
+; CHECK: error: unsupported atomicrmw uinc_wrap: target supports atomics up to 8 bytes, but this atomic accesses 16 bytes
+; CHECK-NOT: __translate_spirv_atomic
+
+ at uv = common dso_local addrspace(1) global <4 x i32> zeroinitializer, align 16
+
+define dso_local spir_func void @oversized_vector() local_unnamed_addr {
+entry:
+ %v = atomicrmw uinc_wrap ptr addrspace(1) @uv, <4 x i32> splat (i32 42) monotonic
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
index f6fd4cf3ed6ab..04e894f0d2263 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-signatures.ll
@@ -1,8 +1,9 @@
; A module may contain uinc_wrap/udec_wrap atomics with mutually incompatible
; signatures, while SPIR-V resolves an imported function by its linkage name
-; alone. Verify that the _p<addrspace>_i<width> suffix keeps them apart: each
+; alone. Verify that the _p<addrspace>_<type> suffix keeps them apart: each
; distinct (address space, value type) combination gets its own declaration,
-; and every call site of a given combination shares that one declaration.
+; and every call site of a given combination shares that one declaration. The
+; value type may be a fixed vector of integers, spelled the LLVM way: v2i32.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
@@ -11,8 +12,10 @@
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#Vec:]] = OpTypeVector %[[#Int]] 2
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
; CHECK-DAG: %[[#Value64:]] = OpConstant %[[#Long]] 42
+; CHECK-DAG: %[[#ValueVec:]] = OpConstantComposite %[[#Vec]] %[[#Value]] %[[#Value]]
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
; The storage class contributes to the memory semantics, so an atomic on a
; Workgroup pointer carries WorkgroupMemory (256) where a CrossWorkgroup one
@@ -23,11 +26,13 @@
; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
; CHECK-DAG: OpDecorate %[[#UIncFnLocal:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p3_i32" Import
; CHECK-DAG: OpDecorate %[[#UIncFn64:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i64" Import
+; CHECK-DAG: OpDecorate %[[#UIncFnVec:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_v2i32" Import
; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
@ui = common dso_local addrspace(1) global i32 0, align 4
@lui = common dso_local addrspace(3) global i32 0, align 4
@ul = common dso_local addrspace(1) global i64 0, align 8
+ at uv = common dso_local addrspace(1) global <2 x i32> zeroinitializer, align 8
; Two atomics of the same value type in different address spaces need two
; incompatible signatures, so they must resolve to two distinct declarations.
@@ -65,3 +70,15 @@ entry:
%b = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
ret void
}
+
+; A vector operand is carried across just like a scalar one, under its own
+; symbol, as long as it fits in the target's maximum atomic size. Anything
+; wider is rejected by AtomicExpandPass before this lowering runs; see
+; atomicrmw-uinc-udec-wrap-oversized.ll.
+
+; CHECK: OpFunctionCall %[[#Vec]] %[[#UIncFnVec]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_Relaxed]] %[[#ValueVec]]
+define dso_local spir_func void @vector_value() local_unnamed_addr {
+entry:
+ %v = atomicrmw uinc_wrap ptr addrspace(1) @uv, <2 x i32> splat (i32 42) monotonic
+ ret void
+}
>From 8ed0c8cab89be151ddb0c7d4d38601965ade09dc Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Tue, 11 Aug 2026 13:16:07 -0500
Subject: [PATCH 03/10] [SPIRV] Keep the metadata on the lowered
uinc_wrap/udec_wrap call
The other memory instructions lowered by this pass copy the metadata of
the instruction they replace. SPIRVCallLowering reads alias.scope and
noalias off the call to build the aliasing decorations and runs after
this pass, so dropping them here loses them silently.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index dc2e8cb30e986..23d93da12ded3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2684,6 +2684,12 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
B.getInt32(MemSem), I.getValOperand()};
CallInst *CI = B.CreateCall(FC, Args);
CI->setCallingConv(CallingConv::SPIR_FUNC);
+ // Keep the metadata, as the other memory instructions lowered here do.
+ // SPIRVCallLowering reads alias.scope/noalias off the call to build the
+ // aliasing decorations, and it runs after this pass, so dropping them here
+ // would silently lose them. insertSpirvDecorations() has already run for the
+ // atomicrmw and does not revisit the call, so nothing is encoded twice.
+ CI->copyMetadata(I);
replaceAllUsesWithAndErase(B, &I, CI);
return CI;
>From fd9e1f759cbf68bcf04dba0cb75043995e04df87 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Tue, 11 Aug 2026 15:02:28 -0500
Subject: [PATCH 04/10] [SPIRV] Update the uinc/udec test for
SPV_KHR_untyped_pointers
AMD-vendor triples enable every valid extension by default
(SPIRVSubtarget), so SPV_KHR_untyped_pointers applies as soon as it
exists. Since 8daf1972106e an ordinary data pointer is spelled
OpTypeUntypedPointerKHR and its variable OpUntypedVariableKHR, carrying
the pointee per instruction. Follow the spelling the extension's own
tests use.
---
llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
index 66c01ef739a60..c8cb0a1a0f075 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -16,11 +16,14 @@
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Bool:]] = OpTypeBool
-; CHECK-DAG: %[[#PointerType:]] = OpTypePointer CrossWorkgroup %[[#Int]]
+; AMD targets enable every valid extension by default, SPV_KHR_untyped_pointers
+; among them, so an ordinary data pointer is untyped here and carries its
+; pointee per instruction instead.
+; CHECK-DAG: %[[#PointerType:]] = OpTypeUntypedPointerKHR CrossWorkgroup
; CHECK-DAG: %[[#MemSem_SequentiallyConsistent:]] = OpConstant %[[#Int]] 528
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
-; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#PointerType]] CrossWorkgroup
+; CHECK-DAG: %[[#Pointer:]] = OpUntypedVariableKHR %[[#PointerType]] CrossWorkgroup %[[#Int]]
; CHECK-DAG: %[[#AllOnes:]] = OpConstant %[[#Int]] 4294967295
; CHECK-DAG: OpDecorate %[[#UIncWrapFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
>From bb18a0db28faef3e2d8da05fc2b6ebab1ec63716 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Mon, 17 Aug 2026 12:08:05 -0500
Subject: [PATCH 05/10] Trim comments.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 21 ++++++-------------
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 5 ++---
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 4 +---
3 files changed, 9 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 23d93da12ded3..541e2d88a77bc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -2633,10 +2633,8 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
return &I;
- // Carrying these across the SPIR-V boundary as a call to an imported helper
- // is an AMD extension: there is no SPIR-V opcode for them, so a consumer has
- // to recognize the helper by name to make sense of the module. Restrict it to
- // AMD targets and let everyone else keep the generic expansion.
+ // No SPIR-V opcode exists for these, so lowering to an imported helper is an
+ // AMD extension. Other targets keep the generic expansion.
if (!isAMDTarget(TM.getTargetTriple()))
return &I;
@@ -2660,13 +2658,9 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
Type *ValTy = I.getValOperand()->getType();
Type *PtrTy = I.getPointerOperand()->getType();
- // Encode the address space and the value type in the name, the same way
- // lowerLLVMIntrinsicName() does for spirv.llvm_memset_p1_i64. A module may
- // need several mutually incompatible signatures, while SPIR-V resolves an
- // imported function by its linkage name alone. The value may also be a fixed
- // vector of integers, spelled the LLVM way: _p1_v2i32. Anything wider than
- // the target's maximum atomic size never reaches here, because AtomicExpand
- // rejects it first.
+ // Encode the address space and value type in the name (e.g. _p1_i32,
+ // _p1_v2i32), as lowerLLVMIntrinsicName() does, since SPIR-V resolves an
+ // imported function by its linkage name alone.
std::string TypeSuffix;
if (auto *VecTy = dyn_cast<FixedVectorType>(ValTy))
TypeSuffix = "v" + std::to_string(VecTy->getNumElements());
@@ -2684,11 +2678,8 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
B.getInt32(MemSem), I.getValOperand()};
CallInst *CI = B.CreateCall(FC, Args);
CI->setCallingConv(CallingConv::SPIR_FUNC);
- // Keep the metadata, as the other memory instructions lowered here do.
// SPIRVCallLowering reads alias.scope/noalias off the call to build the
- // aliasing decorations, and it runs after this pass, so dropping them here
- // would silently lose them. insertSpirvDecorations() has already run for the
- // atomicrmw and does not revisit the call, so nothing is encoded twice.
+ // aliasing decorations and runs after this pass, so preserve the metadata.
CI->copyMetadata(I);
replaceAllUsesWithAndErase(B, &I, CI);
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index e5e1169811bbb..9957764c4af23 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -688,9 +688,8 @@ SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
return AtomicExpansionKind::None;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
- // On AMD targets these are translated into a call to an imported helper by
- // SPIRVEmitIntrinsics, so they must survive to that point unexpanded. Any
- // other target has no such helper and needs the generic expansion.
+ // AMD targets lower these to a helper call in SPIRVEmitIntrinsics, so keep
+ // them unexpanded there; other targets need the generic expansion.
return isAMDTarget(STI.getTargetTriple()) ? AtomicExpansionKind::None
: AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Nand:
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
index c8cb0a1a0f075..80adabc832ac6 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -16,9 +16,7 @@
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
; CHECK-DAG: %[[#Bool:]] = OpTypeBool
-; AMD targets enable every valid extension by default, SPV_KHR_untyped_pointers
-; among them, so an ordinary data pointer is untyped here and carries its
-; pointee per instruction instead.
+; AMD targets enable SPV_KHR_untyped_pointers by default, so pointers are untyped.
; CHECK-DAG: %[[#PointerType:]] = OpTypeUntypedPointerKHR CrossWorkgroup
; CHECK-DAG: %[[#MemSem_SequentiallyConsistent:]] = OpConstant %[[#Int]] 528
; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
>From 34a528a097b874247056c8c8b7b4fc0396663daf Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 04:34:10 -0500
Subject: [PATCH 06/10] [SPIRV] Preserve AMDGPU atomic metadata through
NonSemantic.AuxData
AMDGPU atomic metadata (amdgpu.no.fine.grained.memory,
amdgpu.no.remote.memory, amdgpu.ignore.denormal.mode) was emitted
unconditionally as UserSemantic decorations. UserSemantic is for data
consumers must understand, but this is advisory, so encode it as
NonSemantic.AuxData via a new InstructionMetadata opcode (5), gated on
-spirv-preserve-auxdata. Default output is unchanged.
---
llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp | 7 +--
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp | 32 ++++++++++-
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h | 6 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 22 +++----
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 21 +++++++
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h | 25 ++++++++
.../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 2 +
.../amdgcnspirv-atomic-metadata-decoration.ll | 21 ++++---
...preserve-auxdata-amdgpu-atomic-metadata.ll | 57 +++++++++++++++++++
...-auxdata-atomic-metadata-generic-target.ll | 36 ++++++++++++
10 files changed, 198 insertions(+), 31 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 9f3edf6bac747..d2b1f77257f74 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -887,11 +887,8 @@ void SPIRVAsmPrinter::outputModuleSections() {
MAI = &getAnalysis<SPIRVModuleAnalysis>().MAI;
assert(ST && TII && MAI && M && "Module analysis is required");
- if (!AuxDataHandler) {
- auto Handler = std::make_unique<SPIRVAuxDataHandler>(*this, *M);
- if (Handler->hasWork())
- AuxDataHandler = std::move(Handler);
- }
+ if (!AuxDataHandler && spirvPreserveAuxData())
+ AuxDataHandler = std::make_unique<SPIRVAuxDataHandler>(*this, *M);
// Let the NSDI handler add its extension and ext inst import entry to MAI
// before the module header sections are emitted.
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
index 469c261ac433e..b7af23862d7d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
@@ -63,11 +63,11 @@ SPIRVAuxDataHandler::SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M)
LinkagePreservedGOs.push_back(&GO);
}
-bool SPIRVAuxDataHandler::hasWork() const { return SPVPreserveAuxData; }
+bool llvm::spirvPreserveAuxData() { return SPVPreserveAuxData; }
void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI) {
- if (!hasWork())
+ if (!spirvPreserveAuxData())
return;
if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_non_semantic_info)) {
if (SPVPreserveAuxData)
@@ -76,7 +76,7 @@ void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
return;
}
MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
- if (!MAI.ExtInstSetMap.count(NonSemanticAuxDataSet))
+ if (!MAI.ExtInstSetMap.contains(NonSemanticAuxDataSet))
MAI.ExtInstSetMap[NonSemanticAuxDataSet] = MAI.getNextIDRegister();
}
@@ -181,6 +181,22 @@ void SPIRVAuxDataHandler::emitAuxDataStrings(SPIRV::ModuleAnalysisInfo &MAI) {
collectAttributesFor(&GO, MAI);
collectMetadataFor(&GO, MDNames, MAI);
}
+ // Only a handful of distinct metadata names exist, one per AMDGPUAtomicMDKind
+ // enumerator. Track which we've seen so we can stop once every name has been
+ // emitted, instead of scanning potentially thousands of records with
+ // redundant hash lookups.
+ constexpr unsigned AllMDKindsSeen =
+ (1u << (static_cast<unsigned>(
+ SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind::Last) +
+ 1)) -
+ 1;
+ unsigned SeenMask = 0;
+ for (const auto &Rec : MAI.InstrAuxDataRecords) {
+ SeenMask |= 1u << static_cast<unsigned>(Rec.Kind);
+ getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ if (SeenMask == AllMDKindsSeen)
+ break;
+ }
}
void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
@@ -201,6 +217,16 @@ void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
emitAuxDataExtInst(Rec.Opcode, VoidTypeReg, ExtSetReg, Operands, MAI);
}
+ for (const auto &Rec : MAI.InstrAuxDataRecords) {
+ MCRegister TargetReg = MAI.getRegisterAlias(Rec.MF, Rec.TargetReg);
+ if (!TargetReg.isValid())
+ continue;
+ MCRegister MDNameReg =
+ getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ emitAuxDataExtInst(InstructionMetadataOpcode, VoidTypeReg, ExtSetReg,
+ {TargetReg, MDNameReg}, MAI);
+ }
+
if (LinkagePreservedGOs.empty())
return;
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
index 1d6ba998e6c2e..6ad5c45f6a086 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
@@ -29,7 +29,6 @@ namespace llvm {
class AsmPrinter;
class Constant;
-class Function;
class GlobalObject;
class Module;
class SPIRVSubtarget;
@@ -42,14 +41,13 @@ enum AuxDataOpcode : int64_t {
GlobalVariableMetadataOpcode = 2,
GlobalVariableAttributeOpcode = 3,
LinkageOpcode = 4,
+ InstructionMetadataOpcode = 5,
};
class SPIRVAuxDataHandler {
public:
SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M);
- bool hasWork() const;
-
/// Register extension + ext-inst-set; call before output of section 1.
void prepareModuleOutput(const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI);
@@ -111,6 +109,8 @@ class SPIRVAuxDataHandler {
SPIRV::ModuleAnalysisInfo &MAI);
};
+bool spirvPreserveAuxData();
+
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRVAUXDATAHANDLER_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 541e2d88a77bc..6d4c302ad745c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -13,6 +13,7 @@
#include "SPIRVEmitIntrinsics.h"
#include "SPIRV.h"
+#include "SPIRVAuxDataHandler.h"
#include "SPIRVBuiltins.h"
#include "SPIRVSubtarget.h"
#include "SPIRVTargetMachine.h"
@@ -3081,27 +3082,26 @@ void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
{I->getType()},
{I, MetadataAsValue::get(I->getContext(), MD)});
}
- if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
- isa<AtomicRMWInst>(I)) {
- // If present, we encode AMDGPU atomic metadata as UserSemantic string
- // decorations, which will be parsed during reverse translation.
- auto &Ctx = B.getContext();
- auto *US = ConstantAsMetadata::get(
- ConstantInt::get(B.getInt32Ty(), SPIRV::Decoration::UserSemantic));
+ if (spirvPreserveAuxData() && isa<AtomicRMWInst>(I)) {
+ LLVMContext &Ctx = B.getContext();
+ auto *AuxMD = ConstantAsMetadata::get(ConstantInt::get(
+ B.getInt32Ty(), SPIRV::Decoration::AuxDataInstructionMetadata));
SmallVector<Metadata *> MDs;
if (I->hasMetadata("amdgpu.no.fine.grained.memory"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.no.fine.grained.memory")}));
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.no.fine.grained.memory")}));
if (I->hasMetadata("amdgpu.no.remote.memory"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.no.remote.memory")}));
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.no.remote.memory")}));
if (I->hasMetadata("amdgpu.ignore.denormal.mode"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.ignore.denormal.mode")}));
- if (!MDs.empty())
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.ignore.denormal.mode")}));
+ if (!MDs.empty()) {
+ setInsertPointAfterDef(B, I);
B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()},
{I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDs))});
+ }
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 2818ea3ea81ba..5efcf4b3ac197 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -26,6 +26,7 @@
#include "SPIRVTargetMachine.h"
#include "SPIRVUtils.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -749,6 +750,26 @@ void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
} else if (TII->isAliasingInstr(MI)) {
collectOtherInstr(MI, MAI, SPIRV::MB_AliasingInsts, IS);
} else if (TII->isDecorationInstr(MI)) {
+ if (MI.getOpcode() == SPIRV::OpDecorate &&
+ MI.getOperand(1).getImm() ==
+ static_cast<unsigned>(
+ SPIRV::Decoration::AuxDataInstructionMetadata)) {
+ MAI.setSkipEmission(&MI);
+ std::string Str = getStringImm(MI, 2);
+ using AMDMD = SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind;
+ auto MaybeKind =
+ StringSwitch<std::optional<AMDMD>>(Str)
+ .Case("amdgpu.no.fine.grained.memory",
+ AMDMD::NoFineGrainedMemory)
+ .Case("amdgpu.no.remote.memory", AMDMD::NoRemoteMemory)
+ .Case("amdgpu.ignore.denormal.mode",
+ AMDMD::IgnoreDenormalMode)
+ .Default(std::nullopt);
+ if (MaybeKind)
+ MAI.InstrAuxDataRecords.push_back(
+ {MF, MI.getOperand(0).getReg(), *MaybeKind});
+ continue;
+ }
collectOtherInstr(MI, MAI, SPIRV::MB_Annotations, IS);
collectFuncNames(MI, &F);
} else if (TII->isConstantInstr(MI)) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index 6559b5cfc7457..8114aa63610d1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -167,6 +167,20 @@ struct ModuleAnalysisInfo {
DenseMap<const Function *, SPIRV::FPFastMathDefaultInfoVector>
FPFastMathDefaultInfoMap;
+ enum class AMDGPUAtomicMDKind : uint8_t {
+ NoFineGrainedMemory,
+ NoRemoteMemory,
+ IgnoreDenormalMode,
+
+ Last = IgnoreDenormalMode,
+ };
+ struct InstrAuxDataRecord {
+ const MachineFunction *MF;
+ Register TargetReg;
+ AMDGPUAtomicMDKind Kind;
+ };
+ SmallVector<InstrAuxDataRecord> InstrAuxDataRecords;
+
MCRegister getGlobalObjReg(const GlobalObject *GO) {
assert(GO && "GlobalObject is null");
return GlobalObjMap.lookup(GO);
@@ -211,6 +225,17 @@ struct ModuleAnalysisInfo {
It->second = getNextIDRegister();
return It->second;
}
+ static StringRef getAMDGPUAtomicMDName(AMDGPUAtomicMDKind Kind) {
+ switch (Kind) {
+ case AMDGPUAtomicMDKind::NoFineGrainedMemory:
+ return "amdgpu.no.fine.grained.memory";
+ case AMDGPUAtomicMDKind::NoRemoteMemory:
+ return "amdgpu.no.remote.memory";
+ case AMDGPUAtomicMDKind::IgnoreDenormalMode:
+ return "amdgpu.ignore.denormal.mode";
+ }
+ llvm_unreachable("unknown AMDGPUAtomicMDKind");
+ }
};
} // namespace SPIRV
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 52c174a95cd51..4414b265623fd 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -1437,6 +1437,8 @@ defm FunctionFloatingPointModeINTEL : DecorationOperand<6080, 0, 0, [], [Functio
defm AliasScopeINTEL : DecorationOperand<5914, 0, 0, [], [MemoryAccessAliasingINTEL]>;
defm NoAliasINTEL : DecorationOperand<5915, 0, 0, [], [MemoryAccessAliasingINTEL]>;
defm FPMaxErrorDecorationINTEL : DecorationOperand<6170, 0, 0, [], [FPMaxErrorINTEL]>;
+// Internal-only sentinel; intercepted by SPIRVModuleAnalysis, never emitted.
+defm AuxDataInstructionMetadata : DecorationOperand<0xFFFF, 0, 0, [], []>;
//===----------------------------------------------------------------------===//
// Multiclass used to define BuiltIn enum values and at the same time
diff --git a/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll b/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
index 9c6300f374261..3d679fe7e2281 100644
--- a/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
+++ b/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
@@ -1,18 +1,21 @@
-; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; Without -spirv-preserve-auxdata, AMDGPU atomic metadata must not appear
+; as UserSemantic decorations or any other form in the SPIR-V output.
-; CHECK: OpDecorate %[[#Add:]] UserSemantic "amdgpu.no.fine.grained.memory"
-; CHECK-NEXT: OpDecorate %[[#Add]] UserSemantic "amdgpu.no.remote.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd:]] UserSemantic "amdgpu.no.fine.grained.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd]] UserSemantic "amdgpu.no.remote.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd]] UserSemantic "amdgpu.ignore.denormal.mode"
-; CHECK: %[[#Add]] = OpAtomicIAdd
-; CHECK: %[[#FAdd]] = OpAtomicFAddEXT
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-NOT: amdgpu.no.fine.grained.memory
+; CHECK-NOT: amdgpu.no.remote.memory
+; CHECK-NOT: amdgpu.ignore.denormal.mode
+; CHECK: %[[#Add:]] = OpAtomicIAdd
+; CHECK: %[[#FAdd:]] = OpAtomicFAddEXT
+; CHECK: %[[#Xchg:]] = OpAtomicExchange
define spir_func void @foo(ptr addrspace(1) %p) {
entry:
%atomic.add = atomicrmw add ptr addrspace(1) %p, i32 1 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
%atomic.fadd = atomicrmw fadd ptr addrspace(1) %p, float 1.0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+ %atomic.xchg = atomicrmw xchg ptr addrspace(1) %p, i32 1 seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
new file mode 100644
index 0000000000000..5bd2211db417d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
@@ -0,0 +1,57 @@
+; Test that AMDGPU atomic metadata is preserved as NonSemantic.AuxData
+; InstructionMetadata (opcode 5).
+
+; Positive: with -spirv-preserve-auxdata, metadata emitted as AuxData.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; Negative: without -spirv-preserve-auxdata, no metadata strings.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+; OFF-NOT: amdgpu.no.remote.memory
+; OFF-NOT: amdgpu.ignore.denormal.mode
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
+; CHECK-DAG: %[[#md_idn:]] = OpString "amdgpu.ignore.denormal.mode"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+
+; Integer atomic (add) with two metadata kinds.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res]] %[[#md_nrm]]
+
+; Float atomic (fadd) with all three metadata kinds.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_nrm]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_idn]]
+
+; Atomic (xchg) with only one metadata kind.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#xchg_res:]] %[[#md_nfg]]
+
+; The atomic instructions themselves (forward-referenced by AuxData above).
+; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
+; CHECK-DAG: %[[#fadd_res]] = OpAtomicFAddEXT
+; CHECK-DAG: %[[#xchg_res]] = OpAtomicExchange
+
+
+define amdgpu_kernel void @test_iadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw add ptr addrspace(1) %ptr, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret void
+}
+
+define amdgpu_kernel void @test_fadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw fadd ptr addrspace(1) %ptr, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+ ret void
+}
+
+define amdgpu_kernel void @test_xchg(ptr addrspace(1) %ptr) {
+ %val = atomicrmw xchg ptr addrspace(1) %ptr, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
new file mode 100644
index 0000000000000..9966efaf3482f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
@@ -0,0 +1,36 @@
+; NonSemantic.AuxData is non-semantic by construction: a consumer that does not
+; understand the extended instruction set ignores it. Preserving instruction
+; metadata is therefore not restricted to AMD targets, even though the metadata
+; names that currently benefit are amdgpu.*. Verify that -spirv-preserve-auxdata
+; emits it on a generic target too.
+;
+; The AMD-target behaviour and the full set of metadata kinds are covered by
+; preserve-auxdata-amdgpu-atomic-metadata.ll.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; Without the option nothing is emitted, on any target.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
+
+; No spirv-val run with the option on: the AuxData instruction forward-references
+; the atomic's result <id>, which spirv-val rejects. See the CHECK-INVALID pin in
+; preserve-auxdata-amdgpu-atomic-metadata.ll.
+
+define spir_func void @test_iadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw add ptr addrspace(1) %ptr, i32 1 monotonic, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
>From 83368d59fd9cbd2e90bffc856d43080c5d8f063e Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 07:32:02 -0500
Subject: [PATCH 07/10] [SPIRV] Validate the AuxData atomic-metadata test with
spirv-val
Add a spirv-val run for the default output, and pin the preserve-auxdata
output's current rejection with "not spirv-val": a module-scope forward
reference from InstructionMetadata to an in-function atomic result.
---
...preserve-auxdata-amdgpu-atomic-metadata.ll | 21 +++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
index 5bd2211db417d..5d76ff1965e1b 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
@@ -15,6 +15,27 @@
; OFF-NOT: amdgpu.no.remote.memory
; OFF-NOT: amdgpu.ignore.denormal.mode
+; Default output, with the feature off, validates.
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - -filetype=obj \
+; RUN: | spirv-val %}
+
+; The AuxData instructions go in the module-level section, but the Target
+; operand of InstructionMetadata is the result <id> of an atomic defined inside
+; a function body, so it is a forward reference. That is intentional -- the
+; instruction is non-semantic and its result is never consumed, and
+; NonSemantic.AuxData.asciidoc permits it -- but spirv-val implements no such
+; relaxation and rejects the module. Pin that down so the day validation starts
+; passing is not silent: drop the "not" and the CHECK-INVALID prefix once
+; SPIRV-Tools accepts the forward reference, or once the instructions are moved
+; into the function body after their target.
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - -filetype=obj | not spirv-val 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INVALID %}
+
+; CHECK-INVALID: has not been defined
+
; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
>From 88e3da71d716759a4b16bef1c02fec7d2376d205 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Fri, 7 Aug 2026 07:35:07 -0500
Subject: [PATCH 08/10] [SPIRV] Test AuxData metadata on the
uinc_wrap/udec_wrap lowering
uinc_wrap/udec_wrap lower to an OpFunctionCall, so the AuxData records
attach to the call's result <id>. Add the joint case that neither half's
test covered.
---
...reserve-auxdata-uinc-udec-wrap-metadata.ll | 67 +++++++++++++++++++
1 file changed, 67 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
new file mode 100644
index 0000000000000..1f436f53bea9d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
@@ -0,0 +1,67 @@
+; Test that atomicrmw uinc_wrap/udec_wrap with AMDGPU metadata emit both
+; OpFunctionCall (for the atomic) and AuxData InstructionMetadata (for the
+; metadata), all gated on -spirv-preserve-auxdata.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+; OFF-NOT: amdgpu.no.remote.memory
+
+; Default output, with the feature off, validates.
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 \
+; RUN: -mtriple=spirv64-amd-amdhsa --spirv-ext=+SPV_KHR_non_semantic_info \
+; RUN: %s -o - -filetype=obj | spirv-val %}
+
+; As in preserve-auxdata-amdgpu-atomic-metadata.ll, the AuxData instructions
+; sit in the module-level section and forward-reference a result <id> defined
+; inside a function body -- here the OpFunctionCall standing in for the
+; atomicrmw. spirv-val does not accept that, so pin the rejection rather than
+; leave the module unvalidated. Drop the "not" and the CHECK-INVALID prefix
+; when the forward reference is resolved.
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 \
+; RUN: -mtriple=spirv64-amd-amdhsa --spirv-ext=+SPV_KHR_non_semantic_info \
+; RUN: -spirv-preserve-auxdata %s -o - -filetype=obj | not spirv-val 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INVALID %}
+
+; CHECK-INVALID: has not been defined
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__translate_spirv_atomic_uinc_wrap_p1_i32" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
+
+; AuxData for the uinc_wrap result.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res]] %[[#md_nrm]]
+
+; AuxData for the udec_wrap result.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#udec_res:]] %[[#md_nfg]]
+
+; The function calls themselves.
+; CHECK-DAG: %[[#uinc_res]] = OpFunctionCall %[[#]] %[[#UIncFn]]
+; CHECK-DAG: %[[#udec_res]] = OpFunctionCall %[[#]] %[[#UDecFn]]
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define amdgpu_kernel void @test_uinc_wrap() {
+entry:
+ %uinc = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret void
+}
+
+define amdgpu_kernel void @test_udec_wrap() {
+entry:
+ %udec = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
>From 60dc8c7324b4e8512645156a1b5fd0fd3a33b54e Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Mon, 17 Aug 2026 15:50:48 -0500
Subject: [PATCH 09/10] Address review issues
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 6d4c302ad745c..0be3f2f9058e0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -21,6 +21,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/IRBuilder.h"
@@ -32,6 +33,7 @@
#include "llvm/IR/Value.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
@@ -2653,27 +2655,26 @@ Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
uint32_t MemSem =
static_cast<uint32_t>(getMemSemantics(I.getOrdering())) | ScSem;
- std::string FuncName = (Op == AtomicRMWInst::UIncWrap)
- ? "__translate_spirv_atomic_uinc_wrap"
- : "__translate_spirv_atomic_udec_wrap";
+ SmallString<64> FuncName(Op == AtomicRMWInst::UIncWrap
+ ? "__translate_spirv_atomic_uinc_wrap"
+ : "__translate_spirv_atomic_udec_wrap");
Type *ValTy = I.getValOperand()->getType();
Type *PtrTy = I.getPointerOperand()->getType();
// Encode the address space and value type in the name (e.g. _p1_i32,
// _p1_v2i32), as lowerLLVMIntrinsicName() does, since SPIR-V resolves an
// imported function by its linkage name alone.
- std::string TypeSuffix;
+ raw_svector_ostream OS(FuncName);
+ OS << "_p" << AS << "_";
if (auto *VecTy = dyn_cast<FixedVectorType>(ValTy))
- TypeSuffix = "v" + std::to_string(VecTy->getNumElements());
- TypeSuffix += "i" + std::to_string(ValTy->getScalarSizeInBits());
- FuncName += "_p" + std::to_string(AS) + "_" + TypeSuffix;
+ OS << "v" << VecTy->getNumElements();
+ OS << "i" << ValTy->getScalarSizeInBits();
Type *Int32Ty = B.getInt32Ty();
SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
FunctionType *FT = FunctionType::get(ValTy, ArgTys, false);
FunctionCallee FC = M->getOrInsertFunction(FuncName, FT);
- if (auto *F = dyn_cast<Function>(FC.getCallee()))
- F->setCallingConv(CallingConv::SPIR_FUNC);
+ cast<Function>(FC.getCallee())->setCallingConv(CallingConv::SPIR_FUNC);
SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt32(Scope),
B.getInt32(MemSem), I.getValOperand()};
>From a8ab155d859fee9fe4795d45ef88a07e19494cc9 Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Tue, 18 Aug 2026 10:50:23 -0500
Subject: [PATCH 10/10] Use OpExtInstWithForwardRefsKHR for metadata
preservation.
---
.../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp | 3 +-
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp | 22 +++++++++++--
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h | 4 ++-
llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp | 2 ++
llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 4 +++
.../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 1 +
...preserve-auxdata-amdgpu-atomic-metadata.ll | 32 +++++++++----------
...-auxdata-atomic-metadata-generic-target.ll | 9 ++++--
...reserve-auxdata-uinc-udec-wrap-metadata.ll | 24 +++++++-------
9 files changed, 64 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index 8a2618b5309a9..3d78ac036caa6 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -176,7 +176,8 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
printOpDecorate(MI, OS);
} else if (OpCode == SPIRV::OpExtInstImport) {
recordOpExtInstImport(MI);
- } else if (OpCode == SPIRV::OpExtInst) {
+ } else if (OpCode == SPIRV::OpExtInst ||
+ OpCode == SPIRV::OpExtInstWithForwardRefsKHR) {
printOpExtInst(MI, OS);
} else if (OpCode == SPIRV::UNKNOWN_type) {
printUnknownType(MI, OS);
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
index b7af23862d7d2..a79f94fb3ed2e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
@@ -78,6 +78,18 @@ void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
if (!MAI.ExtInstSetMap.contains(NonSemanticAuxDataSet))
MAI.ExtInstSetMap[NonSemanticAuxDataSet] = MAI.getNextIDRegister();
+
+ // Instruction metadata forward-references its target, so it needs
+ // OpExtInstWithForwardRefsKHR.
+ if (!MAI.InstrAuxDataRecords.empty()) {
+ if (!ST.canUseExtension(
+ SPIRV::Extension::SPV_KHR_relaxed_extended_instruction))
+ report_fatal_error("-spirv-preserve-auxdata with atomic instruction "
+ "metadata requires the "
+ "SPV_KHR_relaxed_extended_instruction extension.");
+ MAI.Reqs.addExtension(
+ SPIRV::Extension::SPV_KHR_relaxed_extended_instruction);
+ }
}
MCRegister
@@ -223,8 +235,10 @@ void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
continue;
MCRegister MDNameReg =
getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ // TargetReg is a function-body result emitted after this section, so it is
+ // always a forward reference.
emitAuxDataExtInst(InstructionMetadataOpcode, VoidTypeReg, ExtSetReg,
- {TargetReg, MDNameReg}, MAI);
+ {TargetReg, MDNameReg}, MAI, /*UseForwardRefs=*/true);
}
if (LinkagePreservedGOs.empty())
@@ -248,9 +262,11 @@ void SPIRVAuxDataHandler::emitAuxDataExtInst(AuxDataOpcode Opcode,
MCRegister VoidTypeReg,
MCRegister ExtSetReg,
ArrayRef<MCRegister> Operands,
- SPIRV::ModuleAnalysisInfo &MAI) {
+ SPIRV::ModuleAnalysisInfo &MAI,
+ bool UseForwardRefs) {
MCInst Inst;
- Inst.setOpcode(SPIRV::OpExtInst);
+ Inst.setOpcode(UseForwardRefs ? SPIRV::OpExtInstWithForwardRefsKHR
+ : SPIRV::OpExtInst);
Inst.addOperand(MCOperand::createReg(MAI.getNextIDRegister()));
Inst.addOperand(MCOperand::createReg(VoidTypeReg));
Inst.addOperand(MCOperand::createReg(ExtSetReg));
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
index 6ad5c45f6a086..3be5a7d030327 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
@@ -104,9 +104,11 @@ class SPIRVAuxDataHandler {
MCRegister emitOpConstantUInt32(uint32_t Value, MCRegister UInt32TypeReg,
SPIRV::ModuleAnalysisInfo &MAI);
MCRegister emitConstant(const Constant *C, SPIRV::ModuleAnalysisInfo &MAI);
+ // UseForwardRefs selects OpExtInstWithForwardRefsKHR instead of OpExtInst.
void emitAuxDataExtInst(AuxDataOpcode Opcode, MCRegister VoidTypeReg,
MCRegister ExtSetReg, ArrayRef<MCRegister> Operands,
- SPIRV::ModuleAnalysisInfo &MAI);
+ SPIRV::ModuleAnalysisInfo &MAI,
+ bool UseForwardRefs = false);
};
bool spirvPreserveAuxData();
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index cee8e5ddfa4f0..ebf5795d19956 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -132,6 +132,8 @@ static const StringMap<SPIRV::Extension::Extension> SPIRVExtensionMap = {
SPIRV::Extension::Extension::SPV_KHR_cooperative_matrix},
{"SPV_KHR_non_semantic_info",
SPIRV::Extension::Extension::SPV_KHR_non_semantic_info},
+ {"SPV_KHR_relaxed_extended_instruction",
+ SPIRV::Extension::Extension::SPV_KHR_relaxed_extended_instruction},
{"SPV_KHR_ray_query", SPIRV::Extension::Extension::SPV_KHR_ray_query},
{"SPV_EXT_shader_image_int64",
SPIRV::Extension::Extension::SPV_EXT_shader_image_int64},
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 12aab2b15d012..4e83bf3f1252b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -152,6 +152,10 @@ def OpExtInstImport: Op<11, (outs ID:$res), (ins StringImm:$extInstsName, variab
// perspective of the instruction selection pass
def OpExtInst: Op<12, (outs ID:$res), (ins TYPE:$ty, i32imm:$set, Extension:$inst, variable_ops),
"$res = OpExtInst $ty $set $inst">;
+// Forward-reference-permitting variant of OpExtInst.
+def OpExtInstWithForwardRefsKHR: Op<4433, (outs ID:$res),
+ (ins TYPE:$ty, i32imm:$set, Extension:$inst, variable_ops),
+ "$res = OpExtInstWithForwardRefsKHR $ty $set $inst">;
// 3.42.5 Mode-Setting Instructions
def OpMemoryModel: Op<14, (outs), (ins AddressingModel:$addr, MemoryModel:$mem),
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 4414b265623fd..99cdc083eeeca 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -405,6 +405,7 @@ defm SPV_AMD_weak_linkage : ExtensionOperand<137, [EnvOpenCL]>;
defm SPV_KHR_abort : ExtensionOperand<138, [EnvOpenCL]>;
defm SPV_KHR_poison_freeze : ExtensionOperand<139, [EnvVulkan, EnvOpenCL]>;
defm SPV_KHR_untyped_pointers : ExtensionOperand<140, [EnvOpenCL]>;
+defm SPV_KHR_relaxed_extended_instruction : ExtensionOperand<141, [EnvVulkan, EnvOpenCL]>;
//===----------------------------------------------------------------------===//
// Multiclass used to define Capabilities enum values and at the same time
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
index 5d76ff1965e1b..8108e4aff9342 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
@@ -3,7 +3,8 @@
; Positive: with -spirv-preserve-auxdata, metadata emitted as AuxData.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
-; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info,+SPV_KHR_relaxed_extended_instruction \
+; RUN: -spirv-preserve-auxdata \
; RUN: %s -o - | FileCheck %s
; Negative: without -spirv-preserve-auxdata, no metadata strings.
@@ -20,22 +21,19 @@
; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - -filetype=obj \
; RUN: | spirv-val %}
-; The AuxData instructions go in the module-level section, but the Target
-; operand of InstructionMetadata is the result <id> of an atomic defined inside
-; a function body, so it is a forward reference. That is intentional -- the
-; instruction is non-semantic and its result is never consumed, and
-; NonSemantic.AuxData.asciidoc permits it -- but spirv-val implements no such
-; relaxation and rejects the module. Pin that down so the day validation starts
-; passing is not silent: drop the "not" and the CHECK-INVALID prefix once
-; SPIRV-Tools accepts the forward reference, or once the instructions are moved
-; into the function body after their target.
+; InstructionMetadata forward-references the atomic's result <id>, encoded as
+; OpExtInstWithForwardRefsKHR. Upstream spirv-val only permits forward refs from
+; debug-info sets, so it still rejects this. Drop the "not"/CHECK-INVALID once
+; https://github.com/KhronosGroup/SPIRV-Tools/pull/6847 lands.
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa \
-; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info,+SPV_KHR_relaxed_extended_instruction \
+; RUN: -spirv-preserve-auxdata \
; RUN: %s -o - -filetype=obj | not spirv-val 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-INVALID %}
; CHECK-INVALID: has not been defined
+; CHECK-DAG: OpExtension "SPV_KHR_relaxed_extended_instruction"
; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
@@ -43,16 +41,16 @@
; CHECK-DAG: %[[#void:]] = OpTypeVoid
; Integer atomic (add) with two metadata kinds.
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res]] %[[#md_nrm]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#add_res]] %[[#md_nrm]]
; Float atomic (fadd) with all three metadata kinds.
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res:]] %[[#md_nfg]]
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_nrm]]
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_idn]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_nrm]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_idn]]
; Atomic (xchg) with only one metadata kind.
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#xchg_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#xchg_res:]] %[[#md_nfg]]
; The atomic instructions themselves (forward-referenced by AuxData above).
; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
index 9966efaf3482f..47369d56462c8 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-atomic-metadata-generic-target.ll
@@ -7,9 +7,11 @@
; The AMD-target behaviour and the full set of metadata kinds are covered by
; preserve-auxdata-amdgpu-atomic-metadata.ll.
+; The forward-referencing metadata needs SPV_KHR_relaxed_extended_instruction,
+; which a generic triple (unlike AMD) does not auto-enable, so request it.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown \
-; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
-; RUN: %s -o - | FileCheck %s
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info,+SPV_KHR_relaxed_extended_instruction \
+; RUN: -spirv-preserve-auxdata %s -o - | FileCheck %s
; Without the option nothing is emitted, on any target.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown \
@@ -18,10 +20,11 @@
; OFF-NOT: amdgpu.no.fine.grained.memory
+; CHECK-DAG: OpExtension "SPV_KHR_relaxed_extended_instruction"
; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
; CHECK-DAG: %[[#void:]] = OpTypeVoid
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
; No spirv-val run with the option on: the AuxData instruction forward-references
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
index 1f436f53bea9d..41bae6e4fc0e9 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
@@ -3,8 +3,8 @@
; metadata), all gated on -spirv-preserve-auxdata.
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
-; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
-; RUN: %s -o - | FileCheck %s
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info,+SPV_KHR_relaxed_extended_instruction \
+; RUN: -spirv-preserve-auxdata %s -o - | FileCheck %s
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
@@ -18,19 +18,19 @@
; RUN: -mtriple=spirv64-amd-amdhsa --spirv-ext=+SPV_KHR_non_semantic_info \
; RUN: %s -o - -filetype=obj | spirv-val %}
-; As in preserve-auxdata-amdgpu-atomic-metadata.ll, the AuxData instructions
-; sit in the module-level section and forward-reference a result <id> defined
-; inside a function body -- here the OpFunctionCall standing in for the
-; atomicrmw. spirv-val does not accept that, so pin the rejection rather than
-; leave the module unvalidated. Drop the "not" and the CHECK-INVALID prefix
-; when the forward reference is resolved.
+; The AuxData forward-reference is encoded as OpExtInstWithForwardRefsKHR, but
+; upstream spirv-val only permits forward refs from debug-info sets, so it still
+; rejects this. Drop the "not"/CHECK-INVALID once
+; https://github.com/KhronosGroup/SPIRV-Tools/pull/6847 lands.
; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 \
-; RUN: -mtriple=spirv64-amd-amdhsa --spirv-ext=+SPV_KHR_non_semantic_info \
+; RUN: -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info,+SPV_KHR_relaxed_extended_instruction \
; RUN: -spirv-preserve-auxdata %s -o - -filetype=obj | not spirv-val 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-INVALID %}
; CHECK-INVALID: has not been defined
+; CHECK-DAG: OpExtension "SPV_KHR_relaxed_extended_instruction"
; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
@@ -40,11 +40,11 @@
; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__translate_spirv_atomic_udec_wrap_p1_i32" Import
; AuxData for the uinc_wrap result.
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res:]] %[[#md_nfg]]
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res]] %[[#md_nrm]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res]] %[[#md_nrm]]
; AuxData for the udec_wrap result.
-; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#udec_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInstWithForwardRefsKHR %[[#void]] %[[#auxset]] {{.+}} %[[#udec_res:]] %[[#md_nfg]]
; The function calls themselves.
; CHECK-DAG: %[[#uinc_res]] = OpFunctionCall %[[#]] %[[#UIncFn]]
More information about the llvm-commits
mailing list