[llvm] [SPIRV] Add __spirv_ builtins for existing instructions (PR #85654)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 18 10:58:38 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-spir-v
Author: Vyacheslav Levytskyy (VyacheslavLevytskyy)
<details>
<summary>Changes</summary>
This PR adds __spirv_ builtins for existing instructions and fixes parsing of "syncscope" values in atomic instructions.
---
Patch is 23.53 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/85654.diff
4 Files Affected:
- (modified) llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp (+55-13)
- (modified) llvm/lib/Target/SPIRV/SPIRVBuiltins.td (+24)
- (modified) llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp (+49-10)
- (added) llvm/test/CodeGen/SPIRV/fence.ll (+54)
``````````diff
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 07be0b34b18271..0478fc33cedc15 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -53,6 +53,8 @@ struct IncomingCall {
: BuiltinName(BuiltinName), Builtin(Builtin),
ReturnRegister(ReturnRegister), ReturnType(ReturnType),
Arguments(Arguments) {}
+
+ bool isSpirvOp() const { return BuiltinName.rfind("__spirv_", 0) == 0; }
};
struct NativeBuiltin {
@@ -485,9 +487,27 @@ static Register buildMemSemanticsReg(Register SemanticsRegister,
return buildConstantIntReg(Semantics, MIRBuilder, GR);
}
+static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode,
+ const SPIRV::IncomingCall *Call,
+ Register TypeReg = Register(0)) {
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ auto MIB = MIRBuilder.buildInstr(Opcode);
+ if (TypeReg.isValid())
+ MIB.addDef(Call->ReturnRegister).addUse(TypeReg);
+ for (Register ArgReg : Call->Arguments) {
+ if (!MRI->getRegClassOrNull(ArgReg))
+ MRI->setRegClass(ArgReg, &SPIRV::IDRegClass);
+ MIB.addUse(ArgReg);
+ }
+ return true;
+}
+
/// Helper function for translating atomic init to OpStore.
static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpStore, Call);
+
assert(Call->Arguments.size() == 2 &&
"Need 2 arguments for atomic init translation");
MIRBuilder.getMRI()->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass);
@@ -502,6 +522,10 @@ static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicLoad, Call, TypeReg);
+
Register PtrRegister = Call->Arguments[0];
MIRBuilder.getMRI()->setRegClass(PtrRegister, &SPIRV::IDRegClass);
// TODO: if true insert call to __translate_ocl_memory_sccope before
@@ -528,7 +552,7 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
.addDef(Call->ReturnRegister)
- .addUse(GR->getSPIRVTypeID(Call->ReturnType))
+ .addUse(TypeReg)
.addUse(PtrRegister)
.addUse(ScopeRegister)
.addUse(MemSemanticsReg);
@@ -539,6 +563,9 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicStore, Call);
+
Register ScopeRegister =
buildConstantIntReg(SPIRV::Scope::Device, MIRBuilder, GR);
Register PtrRegister = Call->Arguments[0];
@@ -557,12 +584,13 @@ static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
}
/// Helper function for building an atomic compare-exchange instruction.
-static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call,
- MachineIRBuilder &MIRBuilder,
- SPIRVGlobalRegistry *GR) {
- const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
- unsigned Opcode =
- SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+static bool buildAtomicCompareExchangeInst(
+ const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin,
+ unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ GR->getSPIRVTypeID(Call->ReturnType));
+
bool IsCmpxchg = Call->Builtin->Name.contains("cmpxchg");
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
@@ -667,6 +695,10 @@ static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ GR->getSPIRVTypeID(Call->ReturnType));
+
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register ScopeRegister =
Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register();
@@ -731,6 +763,12 @@ static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
unsigned Opcode, MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet;
+ Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ IsSet ? TypeReg : Register(0));
+
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register PtrRegister = Call->Arguments[0];
unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent;
@@ -750,9 +788,8 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
buildScopeReg(ScopeRegister, SPIRV::Scope::Device, MIRBuilder, GR, MRI);
auto MIB = MIRBuilder.buildInstr(Opcode);
- if (Opcode == SPIRV::OpAtomicFlagTestAndSet)
- MIB.addDef(Call->ReturnRegister)
- .addUse(GR->getSPIRVTypeID(Call->ReturnType));
+ if (IsSet)
+ MIB.addDef(Call->ReturnRegister).addUse(TypeReg);
MIB.addUse(PtrRegister).addUse(ScopeRegister).addUse(MemSemanticsReg);
return true;
@@ -763,6 +800,9 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call);
+
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
unsigned MemFlags = getIConstVal(Call->Arguments[0], MRI);
unsigned MemSemantics = SPIRV::MemorySemantics::None;
@@ -1240,7 +1280,8 @@ static bool generateAtomicInst(const SPIRV::IncomingCall *Call,
return buildAtomicStoreInst(Call, MIRBuilder, GR);
case SPIRV::OpAtomicCompareExchange:
case SPIRV::OpAtomicCompareExchangeWeak:
- return buildAtomicCompareExchangeInst(Call, MIRBuilder, GR);
+ return buildAtomicCompareExchangeInst(Call, Builtin, Opcode, MIRBuilder,
+ GR);
case SPIRV::OpAtomicIAdd:
case SPIRV::OpAtomicISub:
case SPIRV::OpAtomicOr:
@@ -1815,14 +1856,15 @@ static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call,
SPIRVGlobalRegistry *GR) {
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
const DataLayout &DL = MIRBuilder.getDataLayout();
- bool HasEvents = Call->Builtin->Name.contains("events");
+ bool IsSpirvOp = Call->isSpirvOp();
+ bool HasEvents = Call->Builtin->Name.contains("events") || IsSpirvOp;
const SPIRVType *Int32Ty = GR->getOrCreateSPIRVIntegerType(32, MIRBuilder);
// Make vararg instructions before OpEnqueueKernel.
// Local sizes arguments: Sizes of block invoke arguments. Clang generates
// local size operands as an array, so we need to unpack them.
SmallVector<Register, 16> LocalSizes;
- if (Call->Builtin->Name.find("_varargs") != StringRef::npos) {
+ if (Call->Builtin->Name.find("_varargs") != StringRef::npos || IsSpirvOp) {
const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6;
Register GepReg = Call->Arguments[LocalSizeArrayIdx];
MachineInstr *GepMI = MRI->getUniqueVRegDef(GepReg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index eb26f70b1861f2..ee4f13d89c3c43 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -500,27 +500,38 @@ defm : DemangledNativeBuiltin<"__spirv_All", OpenCL_std, Relational, 1, 1, OpAll
defm : DemangledNativeBuiltin<"atomic_init", OpenCL_std, Atomic, 2, 2, OpStore>;
defm : DemangledNativeBuiltin<"atomic_load", OpenCL_std, Atomic, 1, 1, OpAtomicLoad>;
defm : DemangledNativeBuiltin<"atomic_load_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicLoad>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicLoad", OpenCL_std, Atomic, 3, 3, OpAtomicLoad>;
defm : DemangledNativeBuiltin<"atomic_store", OpenCL_std, Atomic, 2, 2, OpAtomicStore>;
defm : DemangledNativeBuiltin<"atomic_store_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicStore>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicStore", OpenCL_std, Atomic, 4, 4, OpAtomicStore>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicCompareExchange", OpenCL_std, Atomic, 6, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchangeWeak>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchangeWeak>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicCompareExchangeWeak", OpenCL_std, Atomic, 6, 6, OpAtomicCompareExchangeWeak>;
defm : DemangledNativeBuiltin<"atom_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atomic_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atom_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
defm : DemangledNativeBuiltin<"atomic_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicIAdd", OpenCL_std, Atomic, 4, 4, OpAtomicIAdd>;
defm : DemangledNativeBuiltin<"atom_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>;
defm : DemangledNativeBuiltin<"atomic_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicISub", OpenCL_std, Atomic, 4, 4, OpAtomicISub>;
defm : DemangledNativeBuiltin<"atom_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>;
defm : DemangledNativeBuiltin<"atomic_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicOr", OpenCL_std, Atomic, 4, 4, OpAtomicOr>;
defm : DemangledNativeBuiltin<"atom_xor", OpenCL_std, Atomic, 2, 4, OpAtomicXor>;
defm : DemangledNativeBuiltin<"atomic_xor", OpenCL_std, Atomic, 2, 4, OpAtomicXor>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicXor", OpenCL_std, Atomic, 4, 4, OpAtomicXor>;
defm : DemangledNativeBuiltin<"atom_and", OpenCL_std, Atomic, 2, 4, OpAtomicAnd>;
defm : DemangledNativeBuiltin<"atomic_and", OpenCL_std, Atomic, 2, 4, OpAtomicAnd>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicAnd", OpenCL_std, Atomic, 4, 4, OpAtomicAnd>;
defm : DemangledNativeBuiltin<"atomic_exchange", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>;
defm : DemangledNativeBuiltin<"atomic_exchange_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>;
+defm : DemangledNativeBuiltin<"AtomicEx__spirv_change", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>;
defm : DemangledNativeBuiltin<"atomic_work_item_fence", OpenCL_std, Atomic, 1, 3, OpMemoryBarrier>;
+defm : DemangledNativeBuiltin<"__spirv_MemoryBarrier", OpenCL_std, Atomic, 2, 2, OpMemoryBarrier>;
defm : DemangledNativeBuiltin<"atomic_fetch_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
defm : DemangledNativeBuiltin<"atomic_fetch_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>;
defm : DemangledNativeBuiltin<"atomic_fetch_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>;
@@ -532,26 +543,37 @@ defm : DemangledNativeBuiltin<"atomic_fetch_or_explicit", OpenCL_std, Atomic, 4,
defm : DemangledNativeBuiltin<"atomic_fetch_xor_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicXor>;
defm : DemangledNativeBuiltin<"atomic_fetch_and_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicAnd>;
defm : DemangledNativeBuiltin<"atomic_flag_test_and_set", OpenCL_std, Atomic, 1, 1, OpAtomicFlagTestAndSet>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicFlagTestAndSet", OpenCL_std, Atomic, 3, 3, OpAtomicFlagTestAndSet>;
defm : DemangledNativeBuiltin<"atomic_flag_test_and_set_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagTestAndSet>;
defm : DemangledNativeBuiltin<"atomic_flag_clear", OpenCL_std, Atomic, 1, 1, OpAtomicFlagClear>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicFlagClear", OpenCL_std, Atomic, 3, 3, OpAtomicFlagClear>;
defm : DemangledNativeBuiltin<"atomic_flag_clear_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagClear>;
// Barrier builtin records:
defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
+defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>;
// Kernel enqueue builtin records:
defm : DemangledNativeBuiltin<"__enqueue_kernel_basic", OpenCL_std, Enqueue, 5, 5, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"__enqueue_kernel_basic_events", OpenCL_std, Enqueue, 8, 8, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"__enqueue_kernel_varargs", OpenCL_std, Enqueue, 7, 7, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"__enqueue_kernel_events_varargs", OpenCL_std, Enqueue, 10, 10, OpEnqueueKernel>;
+defm : DemangledNativeBuiltin<"__spirv_EnqueueKernel", OpenCL_std, Enqueue, 10, 0, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"retain_event", OpenCL_std, Enqueue, 1, 1, OpRetainEvent>;
+defm : DemangledNativeBuiltin<"__spirv_RetainEvent", OpenCL_std, Enqueue, 1, 1, OpRetainEvent>;
defm : DemangledNativeBuiltin<"release_event", OpenCL_std, Enqueue, 1, 1, OpReleaseEvent>;
+defm : DemangledNativeBuiltin<"__spirv_ReleaseEvent", OpenCL_std, Enqueue, 1, 1, OpReleaseEvent>;
defm : DemangledNativeBuiltin<"create_user_event", OpenCL_std, Enqueue, 0, 0, OpCreateUserEvent>;
+defm : DemangledNativeBuiltin<"__spirv_CreateUserEvent", OpenCL_std, Enqueue, 0, 0, OpCreateUserEvent>;
defm : DemangledNativeBuiltin<"is_valid_event", OpenCL_std, Enqueue, 1, 1, OpIsValidEvent>;
+defm : DemangledNativeBuiltin<"__spirv_IsValidEvent", OpenCL_std, Enqueue, 1, 1, OpIsValidEvent>;
defm : DemangledNativeBuiltin<"set_user_event_status", OpenCL_std, Enqueue, 2, 2, OpSetUserEventStatus>;
+defm : DemangledNativeBuiltin<"__spirv_SetUserEventStatus", OpenCL_std, Enqueue, 2, 2, OpSetUserEventStatus>;
defm : DemangledNativeBuiltin<"capture_event_profiling_info", OpenCL_std, Enqueue, 3, 3, OpCaptureEventProfilingInfo>;
+defm : DemangledNativeBuiltin<"__spirv_CaptureEventProfilingInfo", OpenCL_std, Enqueue, 3, 3, OpCaptureEventProfilingInfo>;
defm : DemangledNativeBuiltin<"get_default_queue", OpenCL_std, Enqueue, 0, 0, OpGetDefaultQueue>;
+defm : DemangledNativeBuiltin<"__spirv_GetDefaultQueue", OpenCL_std, Enqueue, 0, 0, OpGetDefaultQueue>;
defm : DemangledNativeBuiltin<"ndrange_1D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>;
defm : DemangledNativeBuiltin<"ndrange_2D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>;
defm : DemangledNativeBuiltin<"ndrange_3D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>;
@@ -562,7 +584,9 @@ defm : DemangledNativeBuiltin<"__spirv_SpecConstantComposite", OpenCL_std, SpecC
// Async Copy and Prefetch builtin records:
defm : DemangledNativeBuiltin<"async_work_group_copy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>;
+defm : DemangledNativeBuiltin<"__spirv_GroupAsyncCopy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>;
defm : DemangledNativeBuiltin<"wait_group_events", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>;
+defm : DemangledNativeBuiltin<"__spirv_GroupWaitEvents", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>;
// Load and store builtin records:
defm : DemangledNativeBuiltin<"__spirv_Load", OpenCL_std, LoadStore, 1, 3, OpLoad>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 0fef19c2d53419..1f31db65d41464 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -26,10 +26,33 @@
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfoImpls.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/Support/Debug.h"
+namespace llvm {
+
+class SPIRVMachineModuleInfo : public MachineModuleInfoImpl {
+public:
+ SyncScope::ID Work_ItemSSID;
+ SyncScope::ID WorkGroupSSID;
+ SyncScope::ID DeviceSSID;
+ SyncScope::ID AllSVMDevicesSSID;
+ SyncScope::ID SubGroupSSID;
+
+ SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) {
+ LLVMContext &CTX = MMI.getModule()->getContext();
+ Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item");
+ WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup");
+ DeviceSSID = CTX.getOrInsertSyncScopeID("device");
+ AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices");
+ SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group");
+ }
+};
+
+} // end namespace llvm
+
#define DEBUG_TYPE "spirv-isel"
using namespace llvm;
@@ -52,6 +75,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
const RegisterBankInfo &RBI;
SPIRVGlobalRegistry &GR;
MachineRegisterInfo *MRI;
+ SPIRVMachineModuleInfo *MMI = nullptr;
/// We need to keep track of the number we give to anonymous global values to
/// generate the same name every time when this is needed.
@@ -233,6 +257,7 @@ void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
CodeGenCoverage *CoverageInfo,
ProfileSummaryInfo *PSI,
BlockFrequencyInfo *BFI) {
+ MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>();
MRI = &MF.getRegInfo();
GR.setCurrentFunc(MF);
InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
@@ -613,15 +638,27 @@ bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
}
-static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) {
- switch (Ord) {
- case SyncScope::SingleThread:
+static SPIRV::Scope::Scope getScope(SyncScope::ID Ord,
+ SPIRVMachineModuleInfo *MMI) {
+ if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID)
return SPIRV::Scope::Invocation;
- case SyncScope::System:
+ else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID)
+ return SPIRV::Scope::Device;
+ else if (Ord == MMI->WorkGroupSSID)
+ return SPIRV::Scope::Workgroup;
+ else if (Ord == MMI->AllSVMDevicesSSID)
+ return SPIRV::Scope::CrossDevice;
+ else if (Ord == MMI->SubGroupSSID)
+ return SPIRV::Scope::Subgroup;
+ else
+ // OpenCL approach is: "The functions that do not have memory_scope argument
+ // have the same semantics as the corresponding functions with the
+ // memory_scope argument set to memory_scope_device." See ref.: //
+ // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
+ // In our case if the scope is unknown, assuming that SPIR-V code is to be
+ // consumed in an OpenCL environment, we use the same approach and set the
+ // scope to memory_scope_device.
return SPIRV::Scope::Device;
- default:
- llvm_unreachable("Unsupported synchronization Scope ID.");
- }
}
static void addMemoryOperands(MachineMemOperand *MemOp,
@@ -773,7 +810,8 @@ bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
unsigned NegateOpcode) const {
assert(I.hasOneMemOperand());
const MachineMemOperand *MemOp = *I.memoperands_begin();
- uint32_t Scope = static_c...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/85654
More information about the llvm-commits
mailing list