[llvm] [SPIRV] Translate {uinc/udec}_wrap as opaque intrinsics and preserve AMDGPU metadata through NonSemantic.AuxData (PR #213685)
Marcos Maronas via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 07:38:02 PDT 2026
https://github.com/maarquitos14 created https://github.com/llvm/llvm-project/pull/213685
The PR implements an extension to `NonSemantic.AuxData` spec to add `NonSemanticAuxDataInstructionMetadata`, which enables preservation of metadata for instructions other than `GlobalVariable` and `Function` objects for some AMDGPU metadata. Also, it includes the translation of `uinc_wrap` and `udec_wrap` as if they were opaque llvm intrinsics, so that they can be restored in the reverse translation to the exact original code.
>From e2ee6976766412d6c22d9521f1b7f73f6b7efd4f Mon Sep 17 00:00:00 2001
From: Marcos Maronas <mmaronas at amd.com>
Date: Thu, 30 Jul 2026 11:20:52 -0500
Subject: [PATCH] [SPIRV] Translate {uinc/udec}_wrap as opaque intrinsics and
preserve AMDGPU metadata through NonSemantic.AuxData
---
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp | 24 ++++++-
llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h | 4 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 66 ++++++++++++++++---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 22 +++++++
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h | 24 +++++++
.../lib/Target/SPIRV/SPIRVSymbolicOperands.td | 2 +
.../amdgcnspirv-atomic-metadata-decoration.ll | 21 +++---
.../atomicrmw-uinc-udec-wrap-orderings.ll | 66 +++++++++++++++++++
.../SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll | 59 +++++++++++++++++
.../CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll | 47 ++++---------
...preserve-auxdata-amdgpu-atomic-metadata.ll | 57 ++++++++++++++++
...reserve-auxdata-uinc-udec-wrap-metadata.ll | 49 ++++++++++++++
13 files changed, 385 insertions(+), 58 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
create mode 100644 llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
create mode 100644 llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
index 469c261ac433e..c5a10db9685b3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.cpp
@@ -65,6 +65,8 @@ SPIRVAuxDataHandler::SPIRVAuxDataHandler(AsmPrinter &AP, const Module &M)
bool SPIRVAuxDataHandler::hasWork() const { return SPVPreserveAuxData; }
+bool llvm::spirvPreserveAuxData() { return SPVPreserveAuxData; }
+
void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI) {
if (!hasWork())
@@ -76,7 +78,7 @@ void SPIRVAuxDataHandler::prepareModuleOutput(const SPIRVSubtarget &ST,
return;
}
MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
- if (!MAI.ExtInstSetMap.count(NonSemanticAuxDataSet))
+ if (!MAI.ExtInstSetMap.contains(NonSemanticAuxDataSet))
MAI.ExtInstSetMap[NonSemanticAuxDataSet] = MAI.getNextIDRegister();
}
@@ -181,6 +183,16 @@ void SPIRVAuxDataHandler::emitAuxDataStrings(SPIRV::ModuleAnalysisInfo &MAI) {
collectAttributesFor(&GO, MAI);
collectMetadataFor(&GO, MDNames, MAI);
}
+ // There are only 3 possible metadata names (AMDGPUAtomicMDKind has 3
+ // enumerators). Track which we've seen so we can stop early instead of
+ // scanning potentially thousands of records with redundant hash lookups.
+ unsigned SeenMask = 0;
+ for (const auto &Rec : MAI.InstrAuxDataRecords) {
+ SeenMask |= 1u << static_cast<unsigned>(Rec.Kind);
+ getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ if (SeenMask == 0x7)
+ break;
+ }
}
void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
@@ -201,6 +213,16 @@ void SPIRVAuxDataHandler::emitAuxData(SPIRV::ModuleAnalysisInfo &MAI) {
emitAuxDataExtInst(Rec.Opcode, VoidTypeReg, ExtSetReg, Operands, MAI);
}
+ for (const auto &Rec : MAI.InstrAuxDataRecords) {
+ MCRegister TargetReg = MAI.getRegisterAlias(Rec.MF, Rec.TargetReg);
+ if (!TargetReg.isValid())
+ continue;
+ MCRegister MDNameReg =
+ getOrEmitString(MAI.getAMDGPUAtomicMDName(Rec.Kind), MAI);
+ emitAuxDataExtInst(InstructionMetadataOpcode, VoidTypeReg, ExtSetReg,
+ {TargetReg, MDNameReg}, MAI);
+ }
+
if (LinkagePreservedGOs.empty())
return;
diff --git a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
index 1d6ba998e6c2e..cae753bd05f0a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
+++ b/llvm/lib/Target/SPIRV/SPIRVAuxDataHandler.h
@@ -29,7 +29,6 @@ namespace llvm {
class AsmPrinter;
class Constant;
-class Function;
class GlobalObject;
class Module;
class SPIRVSubtarget;
@@ -42,6 +41,7 @@ enum AuxDataOpcode : int64_t {
GlobalVariableMetadataOpcode = 2,
GlobalVariableAttributeOpcode = 3,
LinkageOpcode = 4,
+ InstructionMetadataOpcode = 5,
};
class SPIRVAuxDataHandler {
@@ -111,6 +111,8 @@ class SPIRVAuxDataHandler {
SPIRV::ModuleAnalysisInfo &MAI);
};
+bool spirvPreserveAuxData();
+
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRVAUXDATAHANDLER_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 513b3149dd28f..a71485b2d1932 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -13,6 +13,7 @@
#include "SPIRVEmitIntrinsics.h"
#include "SPIRV.h"
+#include "SPIRVAuxDataHandler.h"
#include "SPIRVBuiltins.h"
#include "SPIRVSubtarget.h"
#include "SPIRVTargetMachine.h"
@@ -394,6 +395,7 @@ class SPIRVEmitIntrinsicsImpl
Instruction *visitStoreInst(StoreInst &I);
Instruction *visitAllocaInst(AllocaInst &I);
Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
Instruction *visitUnreachableInst(UnreachableInst &I);
Instruction *visitCallInst(CallInst &I);
@@ -2609,6 +2611,48 @@ SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
return NewI;
}
+Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
+ auto Op = I.getOperation();
+ if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
+ return &I;
+
+ Module *M = I.getModule();
+ IRBuilder<> B(I.getParent());
+ B.SetInsertPoint(&I);
+
+ const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(*I.getFunction());
+ unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
+
+ uint32_t Scope =
+ static_cast<uint32_t>(getMemScope(I.getContext(), I.getSyncScopeID()));
+ uint32_t ScSem = static_cast<uint32_t>(
+ getMemSemanticsForStorageClass(addressSpaceToStorageClass(AS, ST)));
+ uint32_t MemSem =
+ static_cast<uint32_t>(getMemSemantics(I.getOrdering())) | ScSem;
+
+ StringRef FuncName = (Op == AtomicRMWInst::UIncWrap)
+ ? "__spirv_AtomicUIncWrap"
+ : "__spirv_AtomicUDecWrap";
+
+ Type *ValTy = I.getValOperand()->getType();
+ Type *PtrTy = I.getPointerOperand()->getType();
+ Type *Int32Ty = B.getInt32Ty();
+ SmallVector<Type *, 4> ArgTys = {PtrTy, Int32Ty, Int32Ty, ValTy};
+ FunctionType *FT = FunctionType::get(ValTy, ArgTys, false);
+ FunctionCallee FC = M->getOrInsertFunction(FuncName, FT);
+ if (auto *F = dyn_cast<Function>(FC.getCallee()))
+ F->setCallingConv(CallingConv::SPIR_FUNC);
+
+ SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt32(Scope),
+ B.getInt32(MemSem), I.getValOperand()};
+ CallInst *CI = B.CreateCall(FC, Args);
+ CI->setCallingConv(CallingConv::SPIR_FUNC);
+
+ I.replaceAllUsesWith(CI);
+ I.eraseFromParent();
+ return CI;
+}
+
static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
auto *CI = dyn_cast<CallInst>(&I);
if (!CI)
@@ -3004,27 +3048,29 @@ void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
{I->getType()},
{I, MetadataAsValue::get(I->getContext(), MD)});
}
- if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
+ if (spirvPreserveAuxData() &&
+ I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
isa<AtomicRMWInst>(I)) {
- // If present, we encode AMDGPU atomic metadata as UserSemantic string
- // decorations, which will be parsed during reverse translation.
- auto &Ctx = B.getContext();
- auto *US = ConstantAsMetadata::get(
- ConstantInt::get(B.getInt32Ty(), SPIRV::Decoration::UserSemantic));
+ LLVMContext &Ctx = B.getContext();
+ auto *AuxMD = ConstantAsMetadata::get(ConstantInt::get(
+ B.getInt32Ty(), SPIRV::Decoration::AuxDataInstructionMetadata));
SmallVector<Metadata *> MDs;
if (I->hasMetadata("amdgpu.no.fine.grained.memory"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.no.fine.grained.memory")}));
+ Ctx,
+ {AuxMD, MDString::get(Ctx, "amdgpu.no.fine.grained.memory")}));
if (I->hasMetadata("amdgpu.no.remote.memory"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.no.remote.memory")}));
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.no.remote.memory")}));
if (I->hasMetadata("amdgpu.ignore.denormal.mode"))
MDs.push_back(MDNode::get(
- Ctx, {US, MDString::get(Ctx, "amdgpu.ignore.denormal.mode")}));
- if (!MDs.empty())
+ Ctx, {AuxMD, MDString::get(Ctx, "amdgpu.ignore.denormal.mode")}));
+ if (!MDs.empty()) {
+ setInsertPointAfterDef(B, I);
B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()},
{I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDs))});
+ }
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 64c61d9f6a0c8..3289f724d9313 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -679,7 +679,7 @@ SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
return AtomicExpansionKind::None;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
- return AtomicExpansionKind::CmpXChg;
+ return AtomicExpansionKind::None;
default:
return TargetLowering::shouldExpandAtomicRMWInIR(RMW);
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 2836a2cc43ef0..c30e29651e99e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -26,6 +26,7 @@
#include "SPIRVTargetMachine.h"
#include "SPIRVUtils.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -747,6 +748,27 @@ void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
} else if (TII->isAliasingInstr(MI)) {
collectOtherInstr(MI, MAI, SPIRV::MB_AliasingInsts, IS);
} else if (TII->isDecorationInstr(MI)) {
+ if (MI.getOpcode() == SPIRV::OpDecorate &&
+ MI.getOperand(1).getImm() ==
+ static_cast<unsigned>(
+ SPIRV::Decoration::AuxDataInstructionMetadata)) {
+ MAI.setSkipEmission(&MI);
+ std::string Str = getStringImm(MI, 2);
+ using AMDMD = SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind;
+ auto MaybeKind =
+ StringSwitch<std::optional<AMDMD>>(Str)
+ .Case("amdgpu.no.fine.grained.memory",
+ AMDMD::NoFineGrainedMemory)
+ .Case("amdgpu.no.remote.memory",
+ AMDMD::NoRemoteMemory)
+ .Case("amdgpu.ignore.denormal.mode",
+ AMDMD::IgnoreDenormalMode)
+ .Default(std::nullopt);
+ if (MaybeKind)
+ MAI.InstrAuxDataRecords.push_back(
+ {MF, MI.getOperand(0).getReg(), *MaybeKind});
+ continue;
+ }
collectOtherInstr(MI, MAI, SPIRV::MB_Annotations, IS);
collectFuncNames(MI, &F);
} else if (TII->isConstantInstr(MI)) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index 6559b5cfc7457..a41349805fe8c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -167,6 +167,18 @@ struct ModuleAnalysisInfo {
DenseMap<const Function *, SPIRV::FPFastMathDefaultInfoVector>
FPFastMathDefaultInfoMap;
+ enum class AMDGPUAtomicMDKind : uint8_t {
+ NoFineGrainedMemory,
+ NoRemoteMemory,
+ IgnoreDenormalMode,
+ };
+ struct InstrAuxDataRecord {
+ const MachineFunction *MF;
+ Register TargetReg;
+ AMDGPUAtomicMDKind Kind;
+ };
+ SmallVector<InstrAuxDataRecord> InstrAuxDataRecords;
+
MCRegister getGlobalObjReg(const GlobalObject *GO) {
assert(GO && "GlobalObject is null");
return GlobalObjMap.lookup(GO);
@@ -211,6 +223,18 @@ struct ModuleAnalysisInfo {
It->second = getNextIDRegister();
return It->second;
}
+ static StringRef getAMDGPUAtomicMDName(AMDGPUAtomicMDKind Kind) {
+ switch (Kind) {
+ case AMDGPUAtomicMDKind::NoFineGrainedMemory:
+ return "amdgpu.no.fine.grained.memory";
+ case AMDGPUAtomicMDKind::NoRemoteMemory:
+ return "amdgpu.no.remote.memory";
+ case AMDGPUAtomicMDKind::IgnoreDenormalMode:
+ return "amdgpu.ignore.denormal.mode";
+ default:
+ llvm_unreachable("unknown AMDGPUAtomicMDKind");
+ }
+ }
};
} // namespace SPIRV
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 7da57bc1b2d47..cf4393c4bb884 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -1435,6 +1435,8 @@ defm FunctionFloatingPointModeINTEL : DecorationOperand<6080, 0, 0, [], [Functio
defm AliasScopeINTEL : DecorationOperand<5914, 0, 0, [], [MemoryAccessAliasingINTEL]>;
defm NoAliasINTEL : DecorationOperand<5915, 0, 0, [], [MemoryAccessAliasingINTEL]>;
defm FPMaxErrorDecorationINTEL : DecorationOperand<6170, 0, 0, [], [FPMaxErrorINTEL]>;
+// Internal-only sentinel; intercepted by SPIRVModuleAnalysis, never emitted.
+defm AuxDataInstructionMetadata : DecorationOperand<0xFFFF, 0, 0, [], []>;
//===----------------------------------------------------------------------===//
// Multiclass used to define BuiltIn enum values and at the same time
diff --git a/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll b/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
index 9c6300f374261..3d679fe7e2281 100644
--- a/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
+++ b/llvm/test/CodeGen/SPIRV/amdgcnspirv-atomic-metadata-decoration.ll
@@ -1,18 +1,21 @@
-; RUN: llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+; Without -spirv-preserve-auxdata, AMDGPU atomic metadata must not appear
+; as UserSemantic decorations or any other form in the SPIR-V output.
-; CHECK: OpDecorate %[[#Add:]] UserSemantic "amdgpu.no.fine.grained.memory"
-; CHECK-NEXT: OpDecorate %[[#Add]] UserSemantic "amdgpu.no.remote.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd:]] UserSemantic "amdgpu.no.fine.grained.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd]] UserSemantic "amdgpu.no.remote.memory"
-; CHECK-NEXT: OpDecorate %[[#FAdd]] UserSemantic "amdgpu.ignore.denormal.mode"
-; CHECK: %[[#Add]] = OpAtomicIAdd
-; CHECK: %[[#FAdd]] = OpAtomicFAddEXT
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-NOT: amdgpu.no.fine.grained.memory
+; CHECK-NOT: amdgpu.no.remote.memory
+; CHECK-NOT: amdgpu.ignore.denormal.mode
+; CHECK: %[[#Add:]] = OpAtomicIAdd
+; CHECK: %[[#FAdd:]] = OpAtomicFAddEXT
+; CHECK: %[[#Xchg:]] = OpAtomicExchange
define spir_func void @foo(ptr addrspace(1) %p) {
entry:
%atomic.add = atomicrmw add ptr addrspace(1) %p, i32 1 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
%atomic.fadd = atomicrmw fadd ptr addrspace(1) %p, float 1.0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+ %atomic.xchg = atomicrmw xchg ptr addrspace(1) %p, i32 1 seq_cst, !amdgpu.no.fine.grained.memory !0
ret void
}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
new file mode 100644
index 0000000000000..fb075e9021628
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-orderings.ll
@@ -0,0 +1,66 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode memory
+; orderings in the function call arguments.
+; CrossWorkgroupMemory = 0x200 = 512
+; Monotonic (Relaxed) = 0x000 -> with CrossWorkgroup: 512
+; Acquire = 0x002 -> with CrossWorkgroup: 514
+; Release = 0x004 -> with CrossWorkgroup: 516
+; AcquireRelease = 0x008 -> with CrossWorkgroup: 520
+; SequentiallyConsistent = 0x010 -> with CrossWorkgroup: 528
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#MemSem_Monotonic:]] = OpConstant %[[#Int]] 512
+; CHECK-DAG: %[[#MemSem_Acquire:]] = OpConstant %[[#Int]] 514
+; CHECK-DAG: %[[#MemSem_Release:]] = OpConstant %[[#Int]] 516
+; CHECK-DAG: %[[#MemSem_AcqRel:]] = OpConstant %[[#Int]] 520
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__spirv_AtomicUIncWrap" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__spirv_AtomicUDecWrap" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_orderings() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Monotonic]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 monotonic
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Acquire]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acquire
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_Release]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 release
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_AcqRel]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 acq_rel
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
new file mode 100644
index 0000000000000..b9385d6a4441c
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap-scopes.ll
@@ -0,0 +1,59 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Check that atomicrmw uinc_wrap/udec_wrap correctly encode scopes.
+
+; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
+; CHECK-DAG: %[[#Scope_Device:]] = OpConstant %[[#Int]] 1{{$}}
+; CHECK-DAG: %[[#Scope_Workgroup:]] = OpConstant %[[#Int]] 2{{$}}
+; CHECK-DAG: %[[#Scope_Subgroup:]] = OpConstant %[[#Int]] 3{{$}}
+; CHECK-DAG: %[[#Scope_Invocation:]] = OpConstant %[[#Int]] 4{{$}}
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528{{$}}
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__spirv_AtomicUIncWrap" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__spirv_AtomicUDecWrap" Import
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define dso_local spir_func void @uinc_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UIncFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
+
+define dso_local spir_func void @udec_wrap_scopes() {
+entry:
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]]
+ %0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Device]] %[[#MemSem_SeqCst]]
+ %1 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("device") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Workgroup]] %[[#MemSem_SeqCst]]
+ %2 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("workgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Subgroup]] %[[#MemSem_SeqCst]]
+ %3 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("subgroup") seq_cst
+
+ ; CHECK: OpFunctionCall %[[#Int]] %[[#UDecFn]] %[[#]] %[[#Scope_Invocation]] %[[#MemSem_SeqCst]]
+ %4 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 syncscope("singlethread") seq_cst
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
index 23f8ae6939e86..59f9f0bec9574 100644
--- a/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/SPIRV/atomicrmw-uinc-udec-wrap.ll
@@ -1,55 +1,30 @@
+; Verify that atomicrmw uinc_wrap/udec_wrap lower to OpFunctionCall to
+; __spirv_AtomicUIncWrap/__spirv_AtomicUDecWrap with Import linkage,
+; rather than being expanded to a CmpXChg loop.
+
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
; CHECK-DAG: %[[#Int:]] = OpTypeInt 32 0
-; CHECK-DAG: %[[#Bool:]] = OpTypeBool
-; CHECK-DAG: %[[#PointerType:]] = OpTypePointer CrossWorkgroup %[[#Int]]
-; CHECK-DAG: %[[#MemSem_SequentiallyConsistent:]] = OpConstant %[[#Int]] 528
-; CHECK-DAG: %[[#Value:]] = OpConstant %[[#Int]] 42
-; CHECK-DAG: %[[#One:]] = OpConstant %[[#Int]] 1
+; CHECK-DAG: %[[#MemSem_SeqCst:]] = OpConstant %[[#Int]] 528
+; CHECK-DAG: %[[#WrapVal:]] = OpConstant %[[#Int]] 42
; CHECK-DAG: %[[#Scope_CrossDevice:]] = OpConstantNull %[[#Int]]
-; CHECK-DAG: %[[#Pointer:]] = OpVariable %[[#PointerType]] CrossWorkgroup
- at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK-DAG: OpDecorate %[[#UIncWrapFn:]] LinkageAttributes "__spirv_AtomicUIncWrap" Import
+; CHECK-DAG: OpDecorate %[[#UDecWrapFn:]] LinkageAttributes "__spirv_AtomicUDecWrap" Import
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Add:]] = OpIAdd %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#GE:]] = OpUGreaterThanEqual %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#GE]] %[[#Scope_CrossDevice]] %[[#Add]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+; CHECK: OpFunctionCall %[[#Int]] %[[#UIncWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]] %[[#WrapVal]]
define dso_local spir_func void @atomicrmw_uinc_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst
ret void
}
-; CHECK: %[[#Load:]] = OpLoad %[[#Int]] %[[#Pointer]] Aligned 4
-; CHECK: OpBranch %[[#Loop:]]
-; CHECK: %[[#Loop]] = OpLabel
-; CHECK: %[[#Phi:]] = OpPhi %[[#Int]] %[[#Load]] %[[#Entry:]] %[[#PhiNext:]] %[[#Loop]]
-; CHECK: %[[#Sub:]] = OpISub %[[#Int]] %[[#Phi]] %[[#One]]
-; CHECK: %[[#Equal:]] = OpIEqual %[[#Bool]] %[[#Phi]] %[[#Scope_CrossDevice]]
-; CHECK: %[[#GT:]] = OpUGreaterThan %[[#Bool]] %[[#Phi]] %[[#Value]]
-; CHECK: %[[#Or:]] = OpLogicalOr %[[#Bool]] %[[#Equal]] %[[#GT]]
-; CHECK: %[[#Select:]] = OpSelect %[[#Int]] %[[#Or]] %[[#Value]] %[[#Sub]]
-; CHECK: %[[#CmpXChg:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#Scope_CrossDevice]]
-; CHECK-SAME: %[[#MemSem_SequentiallyConsistent]] %[[#MemSem_SequentiallyConsistent]] %[[#Select]] %[[#Phi]]
-; CHECK: %[[#Cond:]] = OpCompositeExtract %[[#Bool]] %[[#CmpXChgComposite:]] 1
-; CHECK: %[[#PhiNext]] = OpCompositeExtract %[[#Int]] %[[#CmpXChgComposite]] 0
-; CHECK: OpBranchConditional %[[#Cond]] %[[#Exit:]] %[[#Loop]]
-; CHECK: %[[#Exit]] = OpLabel
-
+; CHECK: OpFunctionCall %[[#Int]] %[[#UDecWrapFn]] %[[#]] %[[#Scope_CrossDevice]] %[[#MemSem_SeqCst]] %[[#WrapVal]]
define dso_local spir_func void @atomicrmw_udec_wrap() local_unnamed_addr {
entry:
%0 = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
new file mode 100644
index 0000000000000..5bd2211db417d
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-amdgpu-atomic-metadata.ll
@@ -0,0 +1,57 @@
+; Test that AMDGPU atomic metadata is preserved as NonSemantic.AuxData
+; InstructionMetadata (opcode 5).
+
+; Positive: with -spirv-preserve-auxdata, metadata emitted as AuxData.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; Negative: without -spirv-preserve-auxdata, no metadata strings.
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+; OFF-NOT: amdgpu.no.remote.memory
+; OFF-NOT: amdgpu.ignore.denormal.mode
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
+; CHECK-DAG: %[[#md_idn:]] = OpString "amdgpu.ignore.denormal.mode"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+
+; Integer atomic (add) with two metadata kinds.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#add_res]] %[[#md_nrm]]
+
+; Float atomic (fadd) with all three metadata kinds.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_nrm]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#fadd_res]] %[[#md_idn]]
+
+; Atomic (xchg) with only one metadata kind.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#xchg_res:]] %[[#md_nfg]]
+
+; The atomic instructions themselves (forward-referenced by AuxData above).
+; CHECK-DAG: %[[#add_res]] = OpAtomicIAdd
+; CHECK-DAG: %[[#fadd_res]] = OpAtomicFAddEXT
+; CHECK-DAG: %[[#xchg_res]] = OpAtomicExchange
+
+
+define amdgpu_kernel void @test_iadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw add ptr addrspace(1) %ptr, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret void
+}
+
+define amdgpu_kernel void @test_fadd(ptr addrspace(1) %ptr) {
+ %val = atomicrmw fadd ptr addrspace(1) %ptr, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+ ret void
+}
+
+define amdgpu_kernel void @test_xchg(ptr addrspace(1) %ptr) {
+ %val = atomicrmw xchg ptr addrspace(1) %ptr, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
new file mode 100644
index 0000000000000..729eb76f576fe
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_non_semantic_info/preserve-auxdata-uinc-udec-wrap-metadata.ll
@@ -0,0 +1,49 @@
+; Test that atomicrmw uinc_wrap/udec_wrap with AMDGPU metadata emit both
+; OpFunctionCall (for the atomic) and AuxData InstructionMetadata (for the
+; metadata), all gated on -spirv-preserve-auxdata.
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info -spirv-preserve-auxdata \
+; RUN: %s -o - | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-amd-amdhsa \
+; RUN: --spirv-ext=+SPV_KHR_non_semantic_info %s -o - \
+; RUN: | FileCheck %s --check-prefix=OFF
+
+; OFF-NOT: amdgpu.no.fine.grained.memory
+; OFF-NOT: amdgpu.no.remote.memory
+
+; CHECK-DAG: %[[#auxset:]] = OpExtInstImport "NonSemantic.AuxData"
+; CHECK-DAG: %[[#md_nfg:]] = OpString "amdgpu.no.fine.grained.memory"
+; CHECK-DAG: %[[#md_nrm:]] = OpString "amdgpu.no.remote.memory"
+; CHECK-DAG: %[[#void:]] = OpTypeVoid
+
+; CHECK-DAG: OpDecorate %[[#UIncFn:]] LinkageAttributes "__spirv_AtomicUIncWrap" Import
+; CHECK-DAG: OpDecorate %[[#UDecFn:]] LinkageAttributes "__spirv_AtomicUDecWrap" Import
+
+; AuxData for the uinc_wrap result.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res:]] %[[#md_nfg]]
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#uinc_res]] %[[#md_nrm]]
+
+; AuxData for the udec_wrap result.
+; CHECK-DAG: %[[#]] = OpExtInst %[[#void]] %[[#auxset]] {{.+}} %[[#udec_res:]] %[[#md_nfg]]
+
+; The function calls themselves.
+; CHECK-DAG: %[[#uinc_res]] = OpFunctionCall %[[#]] %[[#UIncFn]]
+; CHECK-DAG: %[[#udec_res]] = OpFunctionCall %[[#]] %[[#UDecFn]]
+
+ at ui = common dso_local addrspace(1) global i32 0, align 4
+
+define amdgpu_kernel void @test_uinc_wrap() {
+entry:
+ %uinc = atomicrmw uinc_wrap ptr addrspace(1) @ui, i32 42 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+ ret void
+}
+
+define amdgpu_kernel void @test_udec_wrap() {
+entry:
+ %udec = atomicrmw udec_wrap ptr addrspace(1) @ui, i32 42 seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+!0 = !{}
More information about the llvm-commits
mailing list