[llvm] c2483ed - [SPIRV] Add __spirv_ builtins for existing instructions (#85654)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 20 11:28:33 PDT 2024
Author: Vyacheslav Levytskyy
Date: 2024-03-20T19:28:29+01:00
New Revision: c2483ed52d6f600a91663a49e35bab1dff2ed977
URL: https://github.com/llvm/llvm-project/commit/c2483ed52d6f600a91663a49e35bab1dff2ed977
DIFF: https://github.com/llvm/llvm-project/commit/c2483ed52d6f600a91663a49e35bab1dff2ed977.diff
LOG: [SPIRV] Add __spirv_ builtins for existing instructions (#85654)
This PR:
* adds __spirv_ builtins for existing instructions;
* fixes parsing of "syncscope" values in atomic instructions;
* fix a special case of binary header emision.
Added:
llvm/test/CodeGen/SPIRV/fence.ll
Modified:
llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
llvm/lib/Target/SPIRV/SPIRVBuiltins.td
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
llvm/test/CodeGen/SPIRV/empty-logical.ll
llvm/test/CodeGen/SPIRV/empty-module.ll
llvm/test/CodeGen/SPIRV/empty-opencl32.ll
llvm/test/CodeGen/SPIRV/empty-opencl64.ll
llvm/test/CodeGen/SPIRV/empty.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 30c67d3fde6338..4eee8062f28248 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -103,22 +103,22 @@ void SPIRVAsmPrinter::emitEndOfAsmFile(Module &M) {
if (ModuleSectionsEmitted == false) {
outputModuleSections();
ModuleSectionsEmitted = true;
- } else {
- ST = static_cast<const SPIRVTargetMachine &>(TM).getSubtargetImpl();
- uint32_t DecSPIRVVersion = ST->getSPIRVVersion();
- uint32_t Major = DecSPIRVVersion / 10;
- uint32_t Minor = DecSPIRVVersion - Major * 10;
- // TODO: calculate Bound more carefully from maximum used register number,
- // accounting for generated OpLabels and other related instructions if
- // needed.
- unsigned Bound = 2 * (ST->getBound() + 1);
- bool FlagToRestore = OutStreamer->getUseAssemblerInfoForParsing();
- OutStreamer->setUseAssemblerInfoForParsing(true);
- if (MCAssembler *Asm = OutStreamer->getAssemblerPtr())
- Asm->setBuildVersion(static_cast<MachO::PlatformType>(0), Major, Minor,
- Bound, VersionTuple(Major, Minor, 0, Bound));
- OutStreamer->setUseAssemblerInfoForParsing(FlagToRestore);
}
+
+ ST = static_cast<const SPIRVTargetMachine &>(TM).getSubtargetImpl();
+ uint32_t DecSPIRVVersion = ST->getSPIRVVersion();
+ uint32_t Major = DecSPIRVVersion / 10;
+ uint32_t Minor = DecSPIRVVersion - Major * 10;
+ // TODO: calculate Bound more carefully from maximum used register number,
+ // accounting for generated OpLabels and other related instructions if
+ // needed.
+ unsigned Bound = 2 * (ST->getBound() + 1);
+ bool FlagToRestore = OutStreamer->getUseAssemblerInfoForParsing();
+ OutStreamer->setUseAssemblerInfoForParsing(true);
+ if (MCAssembler *Asm = OutStreamer->getAssemblerPtr())
+ Asm->setBuildVersion(static_cast<MachO::PlatformType>(0), Major, Minor,
+ Bound, VersionTuple(Major, Minor, 0, Bound));
+ OutStreamer->setUseAssemblerInfoForParsing(FlagToRestore);
}
void SPIRVAsmPrinter::emitFunctionHeader() {
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 07be0b34b18271..0478fc33cedc15 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -53,6 +53,8 @@ struct IncomingCall {
: BuiltinName(BuiltinName), Builtin(Builtin),
ReturnRegister(ReturnRegister), ReturnType(ReturnType),
Arguments(Arguments) {}
+
+ bool isSpirvOp() const { return BuiltinName.rfind("__spirv_", 0) == 0; }
};
struct NativeBuiltin {
@@ -485,9 +487,27 @@ static Register buildMemSemanticsReg(Register SemanticsRegister,
return buildConstantIntReg(Semantics, MIRBuilder, GR);
}
+static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode,
+ const SPIRV::IncomingCall *Call,
+ Register TypeReg = Register(0)) {
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ auto MIB = MIRBuilder.buildInstr(Opcode);
+ if (TypeReg.isValid())
+ MIB.addDef(Call->ReturnRegister).addUse(TypeReg);
+ for (Register ArgReg : Call->Arguments) {
+ if (!MRI->getRegClassOrNull(ArgReg))
+ MRI->setRegClass(ArgReg, &SPIRV::IDRegClass);
+ MIB.addUse(ArgReg);
+ }
+ return true;
+}
+
/// Helper function for translating atomic init to OpStore.
static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpStore, Call);
+
assert(Call->Arguments.size() == 2 &&
"Need 2 arguments for atomic init translation");
MIRBuilder.getMRI()->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass);
@@ -502,6 +522,10 @@ static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicLoad, Call, TypeReg);
+
Register PtrRegister = Call->Arguments[0];
MIRBuilder.getMRI()->setRegClass(PtrRegister, &SPIRV::IDRegClass);
// TODO: if true insert call to __translate_ocl_memory_sccope before
@@ -528,7 +552,7 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)
.addDef(Call->ReturnRegister)
- .addUse(GR->getSPIRVTypeID(Call->ReturnType))
+ .addUse(TypeReg)
.addUse(PtrRegister)
.addUse(ScopeRegister)
.addUse(MemSemanticsReg);
@@ -539,6 +563,9 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicStore, Call);
+
Register ScopeRegister =
buildConstantIntReg(SPIRV::Scope::Device, MIRBuilder, GR);
Register PtrRegister = Call->Arguments[0];
@@ -557,12 +584,13 @@ static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
}
/// Helper function for building an atomic compare-exchange instruction.
-static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call,
- MachineIRBuilder &MIRBuilder,
- SPIRVGlobalRegistry *GR) {
- const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
- unsigned Opcode =
- SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+static bool buildAtomicCompareExchangeInst(
+ const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin,
+ unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ GR->getSPIRVTypeID(Call->ReturnType));
+
bool IsCmpxchg = Call->Builtin->Name.contains("cmpxchg");
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
@@ -667,6 +695,10 @@ static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ GR->getSPIRVTypeID(Call->ReturnType));
+
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register ScopeRegister =
Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register();
@@ -731,6 +763,12 @@ static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call,
static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
unsigned Opcode, MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet;
+ Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType);
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call,
+ IsSet ? TypeReg : Register(0));
+
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
Register PtrRegister = Call->Arguments[0];
unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent;
@@ -750,9 +788,8 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
buildScopeReg(ScopeRegister, SPIRV::Scope::Device, MIRBuilder, GR, MRI);
auto MIB = MIRBuilder.buildInstr(Opcode);
- if (Opcode == SPIRV::OpAtomicFlagTestAndSet)
- MIB.addDef(Call->ReturnRegister)
- .addUse(GR->getSPIRVTypeID(Call->ReturnType));
+ if (IsSet)
+ MIB.addDef(Call->ReturnRegister).addUse(TypeReg);
MIB.addUse(PtrRegister).addUse(ScopeRegister).addUse(MemSemanticsReg);
return true;
@@ -763,6 +800,9 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, Opcode, Call);
+
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
unsigned MemFlags = getIConstVal(Call->Arguments[0], MRI);
unsigned MemSemantics = SPIRV::MemorySemantics::None;
@@ -1240,7 +1280,8 @@ static bool generateAtomicInst(const SPIRV::IncomingCall *Call,
return buildAtomicStoreInst(Call, MIRBuilder, GR);
case SPIRV::OpAtomicCompareExchange:
case SPIRV::OpAtomicCompareExchangeWeak:
- return buildAtomicCompareExchangeInst(Call, MIRBuilder, GR);
+ return buildAtomicCompareExchangeInst(Call, Builtin, Opcode, MIRBuilder,
+ GR);
case SPIRV::OpAtomicIAdd:
case SPIRV::OpAtomicISub:
case SPIRV::OpAtomicOr:
@@ -1815,14 +1856,15 @@ static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call,
SPIRVGlobalRegistry *GR) {
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
const DataLayout &DL = MIRBuilder.getDataLayout();
- bool HasEvents = Call->Builtin->Name.contains("events");
+ bool IsSpirvOp = Call->isSpirvOp();
+ bool HasEvents = Call->Builtin->Name.contains("events") || IsSpirvOp;
const SPIRVType *Int32Ty = GR->getOrCreateSPIRVIntegerType(32, MIRBuilder);
// Make vararg instructions before OpEnqueueKernel.
// Local sizes arguments: Sizes of block invoke arguments. Clang generates
// local size operands as an array, so we need to unpack them.
SmallVector<Register, 16> LocalSizes;
- if (Call->Builtin->Name.find("_varargs") != StringRef::npos) {
+ if (Call->Builtin->Name.find("_varargs") != StringRef::npos || IsSpirvOp) {
const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6;
Register GepReg = Call->Arguments[LocalSizeArrayIdx];
MachineInstr *GepMI = MRI->getUniqueVRegDef(GepReg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index eb26f70b1861f2..ee4f13d89c3c43 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -500,27 +500,38 @@ defm : DemangledNativeBuiltin<"__spirv_All", OpenCL_std, Relational, 1, 1, OpAll
defm : DemangledNativeBuiltin<"atomic_init", OpenCL_std, Atomic, 2, 2, OpStore>;
defm : DemangledNativeBuiltin<"atomic_load", OpenCL_std, Atomic, 1, 1, OpAtomicLoad>;
defm : DemangledNativeBuiltin<"atomic_load_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicLoad>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicLoad", OpenCL_std, Atomic, 3, 3, OpAtomicLoad>;
defm : DemangledNativeBuiltin<"atomic_store", OpenCL_std, Atomic, 2, 2, OpAtomicStore>;
defm : DemangledNativeBuiltin<"atomic_store_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicStore>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicStore", OpenCL_std, Atomic, 4, 4, OpAtomicStore>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicCompareExchange", OpenCL_std, Atomic, 6, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchangeWeak>;
defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchangeWeak>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicCompareExchangeWeak", OpenCL_std, Atomic, 6, 6, OpAtomicCompareExchangeWeak>;
defm : DemangledNativeBuiltin<"atom_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atomic_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
defm : DemangledNativeBuiltin<"atom_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
defm : DemangledNativeBuiltin<"atomic_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicIAdd", OpenCL_std, Atomic, 4, 4, OpAtomicIAdd>;
defm : DemangledNativeBuiltin<"atom_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>;
defm : DemangledNativeBuiltin<"atomic_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicISub", OpenCL_std, Atomic, 4, 4, OpAtomicISub>;
defm : DemangledNativeBuiltin<"atom_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>;
defm : DemangledNativeBuiltin<"atomic_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicOr", OpenCL_std, Atomic, 4, 4, OpAtomicOr>;
defm : DemangledNativeBuiltin<"atom_xor", OpenCL_std, Atomic, 2, 4, OpAtomicXor>;
defm : DemangledNativeBuiltin<"atomic_xor", OpenCL_std, Atomic, 2, 4, OpAtomicXor>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicXor", OpenCL_std, Atomic, 4, 4, OpAtomicXor>;
defm : DemangledNativeBuiltin<"atom_and", OpenCL_std, Atomic, 2, 4, OpAtomicAnd>;
defm : DemangledNativeBuiltin<"atomic_and", OpenCL_std, Atomic, 2, 4, OpAtomicAnd>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicAnd", OpenCL_std, Atomic, 4, 4, OpAtomicAnd>;
defm : DemangledNativeBuiltin<"atomic_exchange", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>;
defm : DemangledNativeBuiltin<"atomic_exchange_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>;
+defm : DemangledNativeBuiltin<"AtomicEx__spirv_change", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>;
defm : DemangledNativeBuiltin<"atomic_work_item_fence", OpenCL_std, Atomic, 1, 3, OpMemoryBarrier>;
+defm : DemangledNativeBuiltin<"__spirv_MemoryBarrier", OpenCL_std, Atomic, 2, 2, OpMemoryBarrier>;
defm : DemangledNativeBuiltin<"atomic_fetch_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
defm : DemangledNativeBuiltin<"atomic_fetch_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>;
defm : DemangledNativeBuiltin<"atomic_fetch_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>;
@@ -532,26 +543,37 @@ defm : DemangledNativeBuiltin<"atomic_fetch_or_explicit", OpenCL_std, Atomic, 4,
defm : DemangledNativeBuiltin<"atomic_fetch_xor_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicXor>;
defm : DemangledNativeBuiltin<"atomic_fetch_and_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicAnd>;
defm : DemangledNativeBuiltin<"atomic_flag_test_and_set", OpenCL_std, Atomic, 1, 1, OpAtomicFlagTestAndSet>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicFlagTestAndSet", OpenCL_std, Atomic, 3, 3, OpAtomicFlagTestAndSet>;
defm : DemangledNativeBuiltin<"atomic_flag_test_and_set_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagTestAndSet>;
defm : DemangledNativeBuiltin<"atomic_flag_clear", OpenCL_std, Atomic, 1, 1, OpAtomicFlagClear>;
+defm : DemangledNativeBuiltin<"__spirv_AtomicFlagClear", OpenCL_std, Atomic, 3, 3, OpAtomicFlagClear>;
defm : DemangledNativeBuiltin<"atomic_flag_clear_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagClear>;
// Barrier builtin records:
defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
+defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>;
// Kernel enqueue builtin records:
defm : DemangledNativeBuiltin<"__enqueue_kernel_basic", OpenCL_std, Enqueue, 5, 5, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"__enqueue_kernel_basic_events", OpenCL_std, Enqueue, 8, 8, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"__enqueue_kernel_varargs", OpenCL_std, Enqueue, 7, 7, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"__enqueue_kernel_events_varargs", OpenCL_std, Enqueue, 10, 10, OpEnqueueKernel>;
+defm : DemangledNativeBuiltin<"__spirv_EnqueueKernel", OpenCL_std, Enqueue, 10, 0, OpEnqueueKernel>;
defm : DemangledNativeBuiltin<"retain_event", OpenCL_std, Enqueue, 1, 1, OpRetainEvent>;
+defm : DemangledNativeBuiltin<"__spirv_RetainEvent", OpenCL_std, Enqueue, 1, 1, OpRetainEvent>;
defm : DemangledNativeBuiltin<"release_event", OpenCL_std, Enqueue, 1, 1, OpReleaseEvent>;
+defm : DemangledNativeBuiltin<"__spirv_ReleaseEvent", OpenCL_std, Enqueue, 1, 1, OpReleaseEvent>;
defm : DemangledNativeBuiltin<"create_user_event", OpenCL_std, Enqueue, 0, 0, OpCreateUserEvent>;
+defm : DemangledNativeBuiltin<"__spirv_CreateUserEvent", OpenCL_std, Enqueue, 0, 0, OpCreateUserEvent>;
defm : DemangledNativeBuiltin<"is_valid_event", OpenCL_std, Enqueue, 1, 1, OpIsValidEvent>;
+defm : DemangledNativeBuiltin<"__spirv_IsValidEvent", OpenCL_std, Enqueue, 1, 1, OpIsValidEvent>;
defm : DemangledNativeBuiltin<"set_user_event_status", OpenCL_std, Enqueue, 2, 2, OpSetUserEventStatus>;
+defm : DemangledNativeBuiltin<"__spirv_SetUserEventStatus", OpenCL_std, Enqueue, 2, 2, OpSetUserEventStatus>;
defm : DemangledNativeBuiltin<"capture_event_profiling_info", OpenCL_std, Enqueue, 3, 3, OpCaptureEventProfilingInfo>;
+defm : DemangledNativeBuiltin<"__spirv_CaptureEventProfilingInfo", OpenCL_std, Enqueue, 3, 3, OpCaptureEventProfilingInfo>;
defm : DemangledNativeBuiltin<"get_default_queue", OpenCL_std, Enqueue, 0, 0, OpGetDefaultQueue>;
+defm : DemangledNativeBuiltin<"__spirv_GetDefaultQueue", OpenCL_std, Enqueue, 0, 0, OpGetDefaultQueue>;
defm : DemangledNativeBuiltin<"ndrange_1D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>;
defm : DemangledNativeBuiltin<"ndrange_2D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>;
defm : DemangledNativeBuiltin<"ndrange_3D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>;
@@ -562,7 +584,9 @@ defm : DemangledNativeBuiltin<"__spirv_SpecConstantComposite", OpenCL_std, SpecC
// Async Copy and Prefetch builtin records:
defm : DemangledNativeBuiltin<"async_work_group_copy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>;
+defm : DemangledNativeBuiltin<"__spirv_GroupAsyncCopy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>;
defm : DemangledNativeBuiltin<"wait_group_events", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>;
+defm : DemangledNativeBuiltin<"__spirv_GroupWaitEvents", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>;
// Load and store builtin records:
defm : DemangledNativeBuiltin<"__spirv_Load", OpenCL_std, LoadStore, 1, 3, OpLoad>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 0fef19c2d53419..5bb8f6084f9671 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -26,10 +26,33 @@
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfoImpls.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/Support/Debug.h"
+namespace llvm {
+
+class SPIRVMachineModuleInfo : public MachineModuleInfoImpl {
+public:
+ SyncScope::ID Work_ItemSSID;
+ SyncScope::ID WorkGroupSSID;
+ SyncScope::ID DeviceSSID;
+ SyncScope::ID AllSVMDevicesSSID;
+ SyncScope::ID SubGroupSSID;
+
+ SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) {
+ LLVMContext &CTX = MMI.getModule()->getContext();
+ Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item");
+ WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup");
+ DeviceSSID = CTX.getOrInsertSyncScopeID("device");
+ AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices");
+ SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group");
+ }
+};
+
+} // end namespace llvm
+
#define DEBUG_TYPE "spirv-isel"
using namespace llvm;
@@ -52,6 +75,7 @@ class SPIRVInstructionSelector : public InstructionSelector {
const RegisterBankInfo &RBI;
SPIRVGlobalRegistry &GR;
MachineRegisterInfo *MRI;
+ SPIRVMachineModuleInfo *MMI = nullptr;
/// We need to keep track of the number we give to anonymous global values to
/// generate the same name every time when this is needed.
@@ -233,6 +257,7 @@ void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
CodeGenCoverage *CoverageInfo,
ProfileSummaryInfo *PSI,
BlockFrequencyInfo *BFI) {
+ MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>();
MRI = &MF.getRegInfo();
GR.setCurrentFunc(MF);
InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
@@ -613,15 +638,27 @@ bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
}
-static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) {
- switch (Ord) {
- case SyncScope::SingleThread:
+static SPIRV::Scope::Scope getScope(SyncScope::ID Ord,
+ SPIRVMachineModuleInfo *MMI) {
+ if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID)
return SPIRV::Scope::Invocation;
- case SyncScope::System:
+ else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID)
+ return SPIRV::Scope::Device;
+ else if (Ord == MMI->WorkGroupSSID)
+ return SPIRV::Scope::Workgroup;
+ else if (Ord == MMI->AllSVMDevicesSSID)
+ return SPIRV::Scope::CrossDevice;
+ else if (Ord == MMI->SubGroupSSID)
+ return SPIRV::Scope::Subgroup;
+ else
+ // OpenCL approach is: "The functions that do not have memory_scope argument
+ // have the same semantics as the corresponding functions with the
+ // memory_scope argument set to memory_scope_device." See ref.: //
+ // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
+ // In our case if the scope is unknown, assuming that SPIR-V code is to be
+ // consumed in an OpenCL environment, we use the same approach and set the
+ // scope to memory_scope_device.
return SPIRV::Scope::Device;
- default:
- llvm_unreachable("Unsupported synchronization Scope ID.");
- }
}
static void addMemoryOperands(MachineMemOperand *MemOp,
@@ -773,7 +810,8 @@ bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
unsigned NegateOpcode) const {
assert(I.hasOneMemOperand());
const MachineMemOperand *MemOp = *I.memoperands_begin();
- uint32_t Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID()));
+ uint32_t Scope =
+ static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI));
Register ScopeReg = buildI32Constant(Scope, I);
Register Ptr = I.getOperand(1).getReg();
@@ -844,7 +882,7 @@ bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
Register MemSemReg = buildI32Constant(MemSem, I);
SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
- uint32_t Scope = static_cast<uint32_t>(getScope(Ord));
+ uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI));
Register ScopeReg = buildI32Constant(Scope, I);
MachineBasicBlock &BB = *I.getParent();
return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
@@ -863,7 +901,8 @@ bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
if (!isa<GIntrinsic>(I)) {
assert(I.hasOneMemOperand());
const MachineMemOperand *MemOp = *I.memoperands_begin();
- unsigned Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID()));
+ unsigned Scope =
+ static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI));
ScopeReg = buildI32Constant(Scope, I);
unsigned ScSem = static_cast<uint32_t>(
@@ -1189,12 +1228,34 @@ bool SPIRVInstructionSelector::selectConstVector(Register ResVReg,
return MIB.constrainAllUses(TII, TRI, RBI);
}
+static unsigned getArrayComponentCount(MachineRegisterInfo *MRI,
+ const SPIRVType *ResType) {
+ Register OpReg = ResType->getOperand(2).getReg();
+ SPIRVType *OpDef = MRI->getVRegDef(OpReg);
+ if (!OpDef)
+ return 0;
+ if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
+ OpDef->getOperand(1).isReg()) {
+ if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
+ OpDef = RefDef;
+ }
+ unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT
+ ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue()
+ : 0;
+ return N;
+}
+
bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
const SPIRVType *ResType,
MachineInstr &I) const {
- if (ResType->getOpcode() != SPIRV::OpTypeVector)
+ unsigned N = 0;
+ if (ResType->getOpcode() == SPIRV::OpTypeVector)
+ N = GR.getScalarOrVectorComponentCount(ResType);
+ else if (ResType->getOpcode() == SPIRV::OpTypeArray)
+ N = getArrayComponentCount(MRI, ResType);
+ else
report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
- unsigned N = GR.getScalarOrVectorComponentCount(ResType);
+
unsigned OpIdx = I.getNumExplicitDefs();
if (!I.getOperand(OpIdx).isReg())
report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
diff --git a/llvm/test/CodeGen/SPIRV/empty-logical.ll b/llvm/test/CodeGen/SPIRV/empty-logical.ll
index a99df5f7eaaa71..1c6604006e2c25 100644
--- a/llvm/test/CodeGen/SPIRV/empty-logical.ll
+++ b/llvm/test/CodeGen/SPIRV/empty-logical.ll
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
;; Ensure the required Capabilities are listed.
; CHECK-DAG: OpCapability Shader
diff --git a/llvm/test/CodeGen/SPIRV/empty-module.ll b/llvm/test/CodeGen/SPIRV/empty-module.ll
index f220176a409905..b56e58ccaf8f1b 100644
--- a/llvm/test/CodeGen/SPIRV/empty-module.ll
+++ b/llvm/test/CodeGen/SPIRV/empty-module.ll
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
; CHECK-DAG: OpCapability Addresses
; CHECK-DAG: OpCapability Linkage
diff --git a/llvm/test/CodeGen/SPIRV/empty-opencl32.ll b/llvm/test/CodeGen/SPIRV/empty-opencl32.ll
index a373781e290bdd..8e826ec35f3781 100644
--- a/llvm/test/CodeGen/SPIRV/empty-opencl32.ll
+++ b/llvm/test/CodeGen/SPIRV/empty-opencl32.ll
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
;; FIXME: ensure Magic Number, version number, generator's magic number, "bound" and "schema" are at least present
diff --git a/llvm/test/CodeGen/SPIRV/empty-opencl64.ll b/llvm/test/CodeGen/SPIRV/empty-opencl64.ll
index d10196525368db..4eaa2e4af0fc76 100644
--- a/llvm/test/CodeGen/SPIRV/empty-opencl64.ll
+++ b/llvm/test/CodeGen/SPIRV/empty-opencl64.ll
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
;; FIXME: ensure Magic Number, version number, generator's magic number, "bound" and "schema" are at least present
diff --git a/llvm/test/CodeGen/SPIRV/empty.ll b/llvm/test/CodeGen/SPIRV/empty.ll
index fdcf316b02566b..390ab329aea33b 100644
--- a/llvm/test/CodeGen/SPIRV/empty.ll
+++ b/llvm/test/CodeGen/SPIRV/empty.ll
@@ -1,4 +1,5 @@
; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
; CHECK: OpCapability Addresses
; CHECK: "foo"
diff --git a/llvm/test/CodeGen/SPIRV/fence.ll b/llvm/test/CodeGen/SPIRV/fence.ll
new file mode 100644
index 00000000000000..5da58667f24f29
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/fence.ll
@@ -0,0 +1,54 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[#GetScope:]] "_Z8getScopev"
+; CHECK-DAG: %[[#Long:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#ScopeDevice:]] = OpConstant %[[#Long]] 1
+; CHECK-DAG: %[[#WrkGrpConst2:]] = OpConstant %[[#Long]] 2
+; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Long]] 3
+; CHECK-DAG: %[[#InvocationConst4:]] = OpConstant %[[#Long]] 4
+; CHECK-DAG: %[[#Const8:]] = OpConstant %[[#Long]] 8
+; CHECK-DAG: %[[#Const16:]] = OpConstant %[[#Long]] 16
+; CHECK-DAG: %[[#Const912:]] = OpConstant %[[#Long]] 912
+; CHECK: OpMemoryBarrier %[[#ScopeDevice]] %[[#WrkGrpConst2]]
+; CHECK: OpMemoryBarrier %[[#ScopeDevice]] %[[#InvocationConst4]]
+; CHECK: OpMemoryBarrier %[[#ScopeDevice]] %[[#Const8]]
+; CHECK: OpMemoryBarrier %[[#InvocationConst4]] %[[#Const16]]
+; CHECK: OpMemoryBarrier %[[#WrkGrpConst2]] %[[#InvocationConst4]]
+; CHECK: OpFunctionEnd
+; CHECK: %[[#ScopeId:]] = OpFunctionCall %[[#Long]] %[[#GetScope]]
+; CHECK: OpControlBarrier %[[#Const3]] %[[#ScopeId:]] %[[#Const912]]
+
+define spir_kernel void @fence_test_kernel1(ptr addrspace(1) noalias %s.ascast) {
+ fence acquire
+ ret void
+}
+
+define spir_kernel void @fence_test_kernel2(ptr addrspace(1) noalias %s.ascast) {
+ fence release
+ ret void
+}
+
+define spir_kernel void @fence_test_kernel3(ptr addrspace(1) noalias %s.ascast) {
+ fence acq_rel
+ ret void
+}
+
+define spir_kernel void @fence_test_kernel4(ptr addrspace(1) noalias %s.ascast) {
+ fence syncscope("singlethread") seq_cst
+ ret void
+}
+
+define spir_kernel void @fence_test_kernel5(ptr addrspace(1) noalias %s.ascast) {
+ fence syncscope("workgroup") release
+ ret void
+}
+
+define spir_func void @barrier_test1() {
+ %scope = call noundef i32 @_Z8getScopev()
+ call void @_Z22__spirv_ControlBarrieriii(i32 noundef 3, i32 noundef %scope, i32 noundef 912)
+ ret void
+}
+
+declare spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef, i32 noundef, i32 noundef)
+declare spir_func i32 @_Z8getScopev()
More information about the llvm-commits
mailing list