[llvm] 8ac46d6 - [SPIR-V] Implement builtins for OpIAddCarry/OpISubBorrow and improve/fix type inference (#115192)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 14 06:30:11 PST 2024
Author: Vyacheslav Levytskyy
Date: 2024-11-14T15:30:05+01:00
New Revision: 8ac46d6b4f8dff07730c4c0dff20d969efcf14f2
URL: https://github.com/llvm/llvm-project/commit/8ac46d6b4f8dff07730c4c0dff20d969efcf14f2
DIFF: https://github.com/llvm/llvm-project/commit/8ac46d6b4f8dff07730c4c0dff20d969efcf14f2.diff
LOG: [SPIR-V] Implement builtins for OpIAddCarry/OpISubBorrow and improve/fix type inference (#115192)
This PR is to solve several intertwined issues with type inference while
adding support for builtins for OpIAddCarry and OpISubBorrow:
* OpIAddCarry and OpISubBorrow generation in a way of supporting SPIR-V
friendly builtins `__spirv_...` -- introduces a new element to account
for, namely, `ptr sret (%struct) %0` argument that is a place to put a
result of the instruction;
* fix early definition of SPIR-V types during call lowering -- namely,
the goal of the PR is to ensure that correct types are applied to
virtual registers which were used as arguments in call lowering and so
caused early definition of SPIR-V types; reproducers are attached as a
new test cases;
* improve parsing of builtin names (e.g., understand a name of a kind
`"anon<int, int> __spirv_IAddCarry<int, int>(int, int)"` that was
incorrectly parsed as `anon` before the PR);
* improve type inference and fix access to erased from parent after
visit instructions -- before the PR visiting of instructions in
emitintrinsics pass replaced old alloca's, bitcast's, etc. instructions
with a newly generated internal SPIR-V intrinsics and after erasing old
instructions there were still references to them in a postprocessing
working list, while records for newly deduced pointee types were lost;
this PR fixes the issue by adding as consistent wrt. internal data
structures action `SPIRVEmitIntrinsics::replaceAllUsesWith()` that fixes
above mentioned problems;
* LLVM IR add/sub instructions result in logical SPIR-V instructions
when applied to bool type;
* fix validation of pointer types for frexp and lgamma_r,
* fix hardcoded reference to AS0 as a Function storage class in
lib/Target/SPIRV/SPIRVBuiltins.cpp -- now it's
`storageClassToAddressSpace(SPIRV::StorageClass::Function)`,
* re-use the same OpTypeStruct for two identical references to struct's
in arithmetic with overflow instructions.
Added:
llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
Modified:
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
llvm/lib/Target/SPIRV/SPIRVBuiltins.td
llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
llvm/lib/Target/SPIRV/SPIRVUtils.cpp
llvm/lib/Target/SPIRV/SPIRVUtils.h
llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 9aa1c31db7d208..06a37f1f559d44 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -190,8 +190,14 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) {
// Check if the extracted name contains type information between angle
// brackets. If so, the builtin is an instantiated template - needs to have
// the information after angle brackets and return type removed.
- if (BuiltinName.find('<') && BuiltinName.back() == '>') {
- BuiltinName = BuiltinName.substr(0, BuiltinName.find('<'));
+ std::size_t Pos1 = BuiltinName.rfind('<');
+ if (Pos1 != std::string::npos && BuiltinName.back() == '>') {
+ std::size_t Pos2 = BuiltinName.rfind(' ', Pos1);
+ if (Pos2 == std::string::npos)
+ Pos2 = 0;
+ else
+ ++Pos2;
+ BuiltinName = BuiltinName.substr(Pos2, Pos1 - Pos2);
BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(' ') + 1);
}
@@ -461,9 +467,11 @@ static Register buildBuiltinVariableLoad(
SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType,
Register Reg = Register(0), bool isConst = true, bool hasLinkageTy = true) {
Register NewRegister =
- MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
- MIRBuilder.getMRI()->setType(NewRegister,
- LLT::pointer(0, GR->getPointerSize()));
+ MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::pIDRegClass);
+ MIRBuilder.getMRI()->setType(
+ NewRegister,
+ LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
+ GR->getPointerSize()));
SPIRVType *PtrType = GR->getOrCreateSPIRVPointerType(
VariableType, MIRBuilder, SPIRV::StorageClass::Input);
GR->assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
@@ -1556,6 +1564,55 @@ static bool generateWaveInst(const SPIRV::IncomingCall *Call,
/* isConst= */ false, /* hasLinkageTy= */ false);
}
+// We expect a builtin
+// Name(ptr sret([RetType]) %result, Type %operand1, Type %operand1)
+// where %result is a pointer to where the result of the builtin execution
+// is to be stored, and generate the following instructions:
+// Res = Opcode RetType Operand1 Operand1
+// OpStore RetVariable Res
+static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+ unsigned Opcode =
+ SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+
+ Register SRetReg = Call->Arguments[0];
+ SPIRVType *PtrRetType = GR->getSPIRVTypeForVReg(SRetReg);
+ SPIRVType *RetType = GR->getPointeeType(PtrRetType);
+ if (!RetType)
+ report_fatal_error("The first parameter must be a pointer");
+ if (RetType->getOpcode() != SPIRV::OpTypeStruct)
+ report_fatal_error("Expected struct type result for the arithmetic with "
+ "overflow builtins");
+
+ SPIRVType *OpType1 = GR->getSPIRVTypeForVReg(Call->Arguments[1]);
+ SPIRVType *OpType2 = GR->getSPIRVTypeForVReg(Call->Arguments[2]);
+ if (!OpType1 || !OpType2 || OpType1 != OpType2)
+ report_fatal_error("Operands must have the same type");
+ if (OpType1->getOpcode() == SPIRV::OpTypeVector)
+ switch (Opcode) {
+ case SPIRV::OpIAddCarryS:
+ Opcode = SPIRV::OpIAddCarryV;
+ break;
+ case SPIRV::OpISubBorrowS:
+ Opcode = SPIRV::OpISubBorrowV;
+ break;
+ }
+
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ Register ResReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+ MRI->setRegClass(ResReg, &SPIRV::iIDRegClass);
+ GR->assignSPIRVTypeToVReg(RetType, ResReg, MIRBuilder.getMF());
+ MIRBuilder.buildInstr(Opcode)
+ .addDef(ResReg)
+ .addUse(GR->getSPIRVTypeID(RetType))
+ .addUse(Call->Arguments[1])
+ .addUse(Call->Arguments[2]);
+ MIRBuilder.buildInstr(SPIRV::OpStore).addUse(SRetReg).addUse(ResReg);
+ return true;
+}
+
static bool generateGetQueryInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
@@ -2511,6 +2568,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
return generateDotOrFMulInst(Call.get(), MIRBuilder, GR);
case SPIRV::Wave:
return generateWaveInst(Call.get(), MIRBuilder, GR);
+ case SPIRV::ICarryBorrow:
+ return generateICarryBorrowInst(Call.get(), MIRBuilder, GR);
case SPIRV::GetQuery:
return generateGetQueryInst(Call.get(), MIRBuilder, GR);
case SPIRV::ImageSizeQuery:
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 29f11c3dd36865..1b95b1479bb932 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -63,6 +63,7 @@ def KernelClock : BuiltinGroup;
def CastToPtr : BuiltinGroup;
def Construct : BuiltinGroup;
def CoopMatr : BuiltinGroup;
+def ICarryBorrow : BuiltinGroup;
//===----------------------------------------------------------------------===//
// Class defining a demangled builtin record. The information in the record
@@ -628,6 +629,10 @@ defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBar
defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>;
+// ICarryBorrow builtin record:
+defm : DemangledNativeBuiltin<"__spirv_IAddCarry", OpenCL_std, ICarryBorrow, 3, 3, OpIAddCarryS>;
+defm : DemangledNativeBuiltin<"__spirv_ISubBorrow", OpenCL_std, ICarryBorrow, 3, 3, OpISubBorrowS>;
+
// cl_intel_split_work_group_barrier
defm : DemangledNativeBuiltin<"intel_work_group_barrier_arrive", OpenCL_std, Barrier, 1, 2, OpControlBarrierArriveINTEL>;
defm : DemangledNativeBuiltin<"__spirv_ControlBarrierArriveINTEL", OpenCL_std, Barrier, 3, 3, OpControlBarrierArriveINTEL>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 98cf598a1f031a..3c5397319aaf21 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -546,12 +546,36 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
ArgVRegs.push_back(ArgReg);
SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg);
if (!SpvType) {
- SpvType = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder);
- GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
+ Type *ArgTy = nullptr;
+ if (auto *PtrArgTy = dyn_cast<PointerType>(Arg.Ty)) {
+ // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we
+ // don't have access to original value in LLVM IR or info about
+ // deduced pointee type, then we should wait with setting the type for
+ // the virtual register until pre-legalizer step when we access
+ // @llvm.spv.assign.ptr.type.p...(...)'s info.
+ if (Arg.OrigValue)
+ if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue))
+ ArgTy =
+ TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
+ } else {
+ ArgTy = Arg.Ty;
+ }
+ if (ArgTy) {
+ SpvType = GR->getOrCreateSPIRVType(ArgTy, MIRBuilder);
+ GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
+ }
}
if (!MRI->getRegClassOrNull(ArgReg)) {
- MRI->setRegClass(ArgReg, GR->getRegClass(SpvType));
- MRI->setType(ArgReg, GR->getRegType(SpvType));
+ // Either we have SpvType created, or Arg.Ty is an untyped pointer and
+ // we know its virtual register's class and type even if we don't know
+ // pointee type.
+ MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
+ : &SPIRV::pIDRegClass);
+ MRI->setType(
+ ArgReg,
+ SpvType ? GR->getRegType(SpvType)
+ : LLT::pointer(cast<PointerType>(Arg.Ty)->getAddressSpace(),
+ GR->getPointerSize()));
}
}
auto instructionSet = canUseOpenCL ? SPIRV::InstructionSet::OpenCL_std
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 8b7e9c48de6c75..e6ef40e010dc20 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -76,8 +76,8 @@ class SPIRVEmitIntrinsics
SPIRV::InstructionSet::InstructionSet InstrSet;
// a register of Instructions that don't have a complete type definition
- SmallPtrSet<Value *, 8> UncompleteTypeInfo;
- SmallVector<Instruction *> PostprocessWorklist;
+ DenseMap<Value *, unsigned> UncompleteTypeInfo;
+ SmallVector<Value *> PostprocessWorklist;
// well known result types of builtins
enum WellKnownTypes { Event };
@@ -147,6 +147,7 @@ class SPIRVEmitIntrinsics
std::unordered_set<Function *> &FVisited);
void replaceWithPtrcasted(Instruction *CI, Type *NewElemTy, Type *KnownElemTy,
CallInst *AssignCI);
+ void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
bool runOnFunction(Function &F);
bool postprocessTypes();
@@ -272,6 +273,27 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
false);
}
+void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
+ bool DeleteOld) {
+ Src->replaceAllUsesWith(Dest);
+ // Update deduced type records
+ GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld);
+ GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld);
+ // Update uncomplete type records if any
+ auto It = UncompleteTypeInfo.find(Src);
+ if (It == UncompleteTypeInfo.end())
+ return;
+ if (DeleteOld) {
+ unsigned Pos = It->second;
+ UncompleteTypeInfo.erase(Src);
+ UncompleteTypeInfo[Dest] = Pos;
+ PostprocessWorklist[Pos] = Dest;
+ } else {
+ UncompleteTypeInfo[Dest] = PostprocessWorklist.size();
+ PostprocessWorklist.push_back(Dest);
+ }
+}
+
static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
isPointerTy(SI->getValueOperand()->getType()) &&
@@ -434,7 +456,7 @@ void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
if (!UnknownElemTypeI8)
return;
if (auto *I = dyn_cast<Instruction>(Op)) {
- UncompleteTypeInfo.insert(I);
+ UncompleteTypeInfo[I] = PostprocessWorklist.size();
PostprocessWorklist.push_back(I);
}
}
@@ -640,7 +662,7 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
if (!UnknownElemTypeI8)
return nullptr;
if (auto *Instr = dyn_cast<Instruction>(I)) {
- UncompleteTypeInfo.insert(Instr);
+ UncompleteTypeInfo[Instr] = PostprocessWorklist.size();
PostprocessWorklist.push_back(Instr);
}
return IntegerType::getInt8Ty(I->getContext());
@@ -1062,7 +1084,7 @@ Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
{I.getOperand(0)->getType()}, {Args});
// remove switch to avoid its unneeded and undesirable unwrap into branches
// and conditions
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
// insert artificial and temporary instruction to preserve valid CFG,
// it will be removed after IR translation pass
@@ -1084,7 +1106,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
for (auto &Op : I.operands())
Args.push_back(Op);
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
return NewI;
}
@@ -1099,7 +1121,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
// such bitcasts do not provide sufficient information, should be just skipped
// here, and handled in insertPtrCastOrAssignTypeInstr.
if (isPointerTy(I.getType())) {
- I.replaceAllUsesWith(Source);
+ replaceAllUsesWith(&I, Source);
I.eraseFromParent();
return nullptr;
}
@@ -1108,7 +1130,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
SmallVector<Value *> Args(I.op_begin(), I.op_end());
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1219,6 +1241,8 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(AddressSpace)};
auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
I->setOperand(OperandToReplace, PtrCastI);
+ // We need to set up a pointee type for the newly created spv_ptrcast.
+ buildAssignPtr(B, ExpectedElementType, PtrCastI);
}
void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
@@ -1331,7 +1355,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
SmallVector<Value *> Args(I.op_begin(), I.op_end());
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1346,7 +1370,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1382,7 +1406,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
Args.push_back(B.getInt32(Op));
auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
return NewI;
}
@@ -1443,7 +1467,7 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
{PtrTy, ArraySize->getType()}, {ArraySize})
: B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1613,7 +1637,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
auto *NewOp =
buildIntrWithMD(Intrinsic::spv_track_constant,
{II->getType(), II->getType()}, t->second, I, {}, B);
- I->replaceAllUsesWith(NewOp);
+ replaceAllUsesWith(I, NewOp, false);
NewOp->setArgOperand(0, I);
}
bool IsPhi = isa<PHINode>(I), BPrepared = false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 7dfe8ef6366f9b..3bb86e8be69500 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -180,6 +180,15 @@ class SPIRVGlobalRegistry {
auto It = AssignPtrTypeInstr.find(Val);
return It == AssignPtrTypeInstr.end() ? nullptr : It->second;
}
+ // - Find a record and update its key or add a new record, if found.
+ void updateIfExistAssignPtrTypeInstr(Value *OldVal, Value *NewVal,
+ bool DeleteOld) {
+ if (CallInst *CI = findAssignPtrTypeInstr(OldVal)) {
+ if (DeleteOld)
+ AssignPtrTypeInstr.erase(OldVal);
+ AssignPtrTypeInstr[NewVal] = CI;
+ }
+ }
// A registry of mutated values
// (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`):
@@ -214,6 +223,15 @@ class SPIRVGlobalRegistry {
auto It = DeducedElTys.find(Val);
return It == DeducedElTys.end() ? nullptr : It->second;
}
+ // - Find a record and update its key or add a new record, if found.
+ void updateIfExistDeducedElementType(Value *OldVal, Value *NewVal,
+ bool DeleteOld) {
+ if (Type *Ty = findDeducedElementType(OldVal)) {
+ if (DeleteOld)
+ DeducedElTys.erase(OldVal);
+ DeducedElTys[NewVal] = Ty;
+ }
+ }
// - Add a record to the map of deduced composite types.
void addDeducedCompositeType(Value *Val, Type *Ty) {
DeducedNestedTys[Val] = Ty;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index ecbceb5b472fa1..59a1bf50b771b9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -414,6 +414,17 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
validateForwardCalls(STI, MRI, GR, MI);
break;
+ // ensure that LLVM IR add/sub instructions result in logical SPIR-V
+ // instructions when applied to bool type
+ case SPIRV::OpIAddS:
+ case SPIRV::OpIAddV:
+ case SPIRV::OpISubS:
+ case SPIRV::OpISubV:
+ if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
+ SPIRV::OpTypeBool))
+ MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
+ break;
+
// ensure that LLVM IR bitwise instructions result in logical SPIR-V
// instructions when applied to bool type
case SPIRV::OpBitwiseOrS:
@@ -473,8 +484,11 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)
continue;
switch (MI.getOperand(3).getImm()) {
+ case SPIRV::OpenCLExtInst::frexp:
+ case SPIRV::OpenCLExtInst::lgamma_r:
case SPIRV::OpenCLExtInst::remquo: {
- // The last operand must be of a pointer to the return type.
+ // The last operand must be of a pointer to i32 or vector of i32
+ // values.
MachineIRBuilder MIB(MI);
SPIRVType *Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB);
SPIRVType *RetType = MRI->getVRegDef(MI.getOperand(1).getReg());
@@ -487,8 +501,6 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
Int32Type, RetType->getOperand(2).getImm(), MIB));
} break;
case SPIRV::OpenCLExtInst::fract:
- case SPIRV::OpenCLExtInst::frexp:
- case SPIRV::OpenCLExtInst::lgamma_r:
case SPIRV::OpenCLExtInst::modf:
case SPIRV::OpenCLExtInst::sincos:
// The last operand must be of a pointer to the base type represented
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index b5dce14cd0bc1e..158314d23393ae 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1191,7 +1191,7 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
// "Result Type must be from OpTypeStruct. The struct must have two members,
// and the two members must be the same type."
Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
- ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy});
+ ResTy = StructType::get(ResElemTy, ResElemTy);
// Build SPIR-V types and constant(s) if needed.
MachineIRBuilder MIRBuilder(I);
SPIRVType *StructType = GR.getOrCreateSPIRVType(
@@ -3178,22 +3178,9 @@ bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
MachineInstr &I) const {
// Change order of instructions if needed: all OpVariable instructions in a
// function must be the first instructions in the first block
- MachineFunction *MF = I.getParent()->getParent();
- MachineBasicBlock *MBB = &MF->front();
- auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end();
- bool IsHeader = false;
- unsigned Opcode;
- for (; It != E && It != I; ++It) {
- Opcode = It->getOpcode();
- if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
- IsHeader = true;
- } else if (IsHeader &&
- !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
- ++It;
- break;
- }
- }
- return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
+ auto It = getOpVariableMBBIt(I);
+ return BuildMI(*It->getParent(), It, It->getDebugLoc(),
+ TII.get(SPIRV::OpVariable))
.addDef(ResVReg)
.addUse(GR.getSPIRVTypeID(ResType))
.addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index f9b361e163c909..aeb2c29f7b8618 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -162,6 +162,26 @@ void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
}
}
+MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I) {
+ MachineFunction *MF = I.getParent()->getParent();
+ MachineBasicBlock *MBB = &MF->front();
+ MachineBasicBlock::iterator It = MBB->SkipPHIsAndLabels(MBB->begin()),
+ E = MBB->end();
+ bool IsHeader = false;
+ unsigned Opcode;
+ for (; It != E && It != I; ++It) {
+ Opcode = It->getOpcode();
+ if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
+ IsHeader = true;
+ } else if (IsHeader &&
+ !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
+ ++It;
+ break;
+ }
+ }
+ return It;
+}
+
SPIRV::StorageClass::StorageClass
addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
switch (AddrSpace) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 11fd3a5c61dcae..298b0b93b0e4d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -15,6 +15,7 @@
#include "MCTargetDesc/SPIRVBaseInfo.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/TypedPointerType.h"
@@ -139,6 +140,10 @@ void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
const MDNode *GVarMD);
+// Return a valid position for the OpVariable instruction inside a function,
+// i.e., at the beginning of the first block of the function.
+MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I);
+
// Convert a SPIR-V storage class to the corresponding LLVM IR address space.
// TODO: maybe the following two functions should be handled in the subtarget
// to allow for
diff erent OpenCL vs Vulkan handling.
diff --git a/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
new file mode 100644
index 00000000000000..8f14eba21b63a2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
@@ -0,0 +1,129 @@
+; Adapted from Khronos Translator test suite: test/iaddcarry_builtin.ll
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%i8struct = type {i8, i8}
+%i16struct = type {i16, i16}
+%i32struct = type {i32, i32}
+%i64struct = type {i64, i64}
+%vecstruct = type {<4 x i32>, <4 x i32>}
+
+; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
+; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
+; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32
+; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
+; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
+; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
+; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
+; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
+; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
+; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
+; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
+; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
+
+define spir_func void @test_builtin_iaddcarrycc(i8 %a, i8 %b) {
+ entry:
+ %0 = alloca %i8struct
+ call void @_Z17__spirv_IAddCarrycc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
+; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpIAddCarry [[i8struct]] [[a]] [[b]]
+; CHECK-SPIRV: OpStore [[var_11]] [[var_12]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) {
+ entry:
+ %0 = alloca %i16struct
+ call void @_Z17__spirv_IAddCarryss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
+; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpIAddCarry [[i16struct]] [[a_0]] [[b_0]]
+; CHECK-SPIRV: OpStore [[var_21]] [[var_22]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %i32struct
+ call void @_Z17__spirv_IAddCarryii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
+; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpIAddCarry [[i32struct]] [[a_1]] [[b_1]]
+; CHECK-SPIRV: OpStore [[var_31]] [[var_32]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) {
+ entry:
+ %0 = alloca %i64struct
+ call void @_Z17__spirv_IAddCarryll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
+; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpIAddCarry [[i64struct]] [[a_2]] [[b_2]]
+; CHECK-SPIRV: OpStore [[var_41]] [[var_42]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
+ entry:
+ %0 = alloca %vecstruct
+ call void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
+; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpIAddCarry [[vecstruct]] [[a_3]] [[b_3]]
+; CHECK-SPIRV: OpStore [[var_51]] [[var_52]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+%struct.anon = type { i32, i32 }
+
+define spir_func void @test_builtin_iaddcarry_anon(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %struct.anon
+ %1 = addrspacecast ptr %0 to ptr addrspace(4)
+ call spir_func void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
+; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
+; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpIAddCarry [[struct_anon]] [[a_4]] [[b_4]]
+; CHECK-SPIRV: OpStore [[var_61]] [[var_62]]
+
+declare void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
+declare void @_Z17__spirv_IAddCarrycc(ptr sret(%i8struct), i8, i8)
+declare void @_Z17__spirv_IAddCarryss(ptr sret(%i16struct), i16, i16)
+declare void @_Z17__spirv_IAddCarryii(ptr sret(%i32struct), i32, i32)
+declare void @_Z17__spirv_IAddCarryll(ptr sret(%i64struct), i64, i64)
+declare void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)
diff --git a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
index d222dfa570cf78..411866a67cc67b 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
@@ -1,5 +1,11 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; CHECK-DAG: OpName [[BOOL_ADD:%.+]] "bool_add"
+; CHECK-DAG: OpName [[BOOL_SUB:%.+]] "bool_sub"
; CHECK-DAG: OpName [[SCALAR_ADD:%.+]] "scalar_add"
; CHECK-DAG: OpName [[SCALAR_SUB:%.+]] "scalar_sub"
; CHECK-DAG: OpName [[SCALAR_MUL:%.+]] "scalar_mul"
@@ -10,13 +16,28 @@
; CHECK-NOT: DAG-FENCE
+; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
; CHECK-DAG: [[SCALAR:%.+]] = OpTypeInt 32
; CHECK-DAG: [[SCALAR_FN:%.+]] = OpTypeFunction [[SCALAR]] [[SCALAR]] [[SCALAR]]
+; CHECK-DAG: [[BOOL_FN:%.+]] = OpTypeFunction [[BOOL]] [[BOOL]] [[BOOL]]
; CHECK-NOT: DAG-FENCE
;; Test add on scalar:
+define i1 @bool_add(i1 %a, i1 %b) {
+ %c = add i1 %a, %b
+ ret i1 %c
+}
+
+; CHECK: [[BOOL_ADD]] = OpFunction [[BOOL]] None [[BOOL_FN]]
+; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK: OpLabel
+; CHECK: [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]]
+; CHECK: OpReturnValue [[C]]
+; CHECK-NEXT: OpFunctionEnd
+
define i32 @scalar_add(i32 %a, i32 %b) {
%c = add i32 %a, %b
ret i32 %c
@@ -32,6 +53,19 @@ define i32 @scalar_add(i32 %a, i32 %b) {
;; Test sub on scalar:
+define i1 @bool_sub(i1 %a, i1 %b) {
+ %c = sub i1 %a, %b
+ ret i1 %c
+}
+
+; CHECK: [[BOOL_SUB]] = OpFunction [[BOOL]] None [[BOOL_FN]]
+; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK: OpLabel
+; CHECK: [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]]
+; CHECK: OpReturnValue [[C]]
+; CHECK-NEXT: OpFunctionEnd
+
define i32 @scalar_sub(i32 %a, i32 %b) {
%c = sub i32 %a, %b
ret i32 %c
diff --git a/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
new file mode 100644
index 00000000000000..08b4d2a1fa8e55
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
@@ -0,0 +1,127 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%i8struct = type {i8, i8}
+%i16struct = type {i16, i16}
+%i32struct = type {i32, i32}
+%i64struct = type {i64, i64}
+%vecstruct = type {<4 x i32>, <4 x i32>}
+
+; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
+; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
+; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32
+; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
+; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
+; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
+; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
+; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
+; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
+; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
+; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
+; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
+
+define spir_func void @test_builtin_isubborrowcc(i8 %a, i8 %b) {
+ entry:
+ %0 = alloca %i8struct
+ call void @_Z18__spirv_ISubBorrowcc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
+; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpISubBorrow [[i8struct]] [[a]] [[b]]
+; CHECK-SPIRV: OpStore [[var_11]] [[var_12]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowss(i16 %a, i16 %b) {
+ entry:
+ %0 = alloca %i16struct
+ call void @_Z18__spirv_ISubBorrowss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
+; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpISubBorrow [[i16struct]] [[a_0]] [[b_0]]
+; CHECK-SPIRV: OpStore [[var_21]] [[var_22]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowii(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %i32struct
+ call void @_Z18__spirv_ISubBorrowii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
+; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpISubBorrow [[i32struct]] [[a_1]] [[b_1]]
+; CHECK-SPIRV: OpStore [[var_31]] [[var_32]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowll(i64 %a, i64 %b) {
+ entry:
+ %0 = alloca %i64struct
+ call void @_Z18__spirv_ISubBorrowll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
+; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpISubBorrow [[i64struct]] [[a_2]] [[b_2]]
+; CHECK-SPIRV: OpStore [[var_41]] [[var_42]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
+ entry:
+ %0 = alloca %vecstruct
+ call void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
+; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpISubBorrow [[vecstruct]] [[a_3]] [[b_3]]
+; CHECK-SPIRV: OpStore [[var_51]] [[var_52]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+%struct.anon = type { i32, i32 }
+
+define spir_func void @test_builtin_isubborrow_anon(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %struct.anon
+ %1 = addrspacecast ptr %0 to ptr addrspace(4)
+ call spir_func void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
+; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
+; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpISubBorrow [[struct_anon]] [[a_4]] [[b_4]]
+; CHECK-SPIRV: OpStore [[var_61]] [[var_62]]
+
+declare void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
+declare void @_Z18__spirv_ISubBorrowcc(ptr sret(%i8struct), i8, i8)
+declare void @_Z18__spirv_ISubBorrowss(ptr sret(%i16struct), i16, i16)
+declare void @_Z18__spirv_ISubBorrowii(ptr sret(%i32struct), i32, i32)
+declare void @_Z18__spirv_ISubBorrowll(ptr sret(%i64struct), i64, i64)
+declare void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
index cecd6f60655dcf..1c77800a1630d5 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
@@ -20,6 +20,7 @@
; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+; CHECK-DAG: %[[StructLong:.*]] = OpTypeStruct %[[Long]] %[[Long]]
; CHECK: OpFunction
; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
@@ -84,6 +85,20 @@ define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p)
ret void
}
+; This is to check that we re-use the same OpTypeStruct for two identical references to { i64, i1 }.
+; CHECK: OpFunction
+; CHECK: OpIAddCarry %[[StructLong]]
+; CHECK: OpIAddCarry %[[StructLong]]
+; CHECK: OpReturn
+define void @foo(i64 %a, i64 %b) {
+ %r1 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b)
+ %r2 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b)
+ %d1 = extractvalue { i64, i1 } %r1, 0
+ %d2 = extractvalue { i64, i1 } %r2, 0
+ ret void
+}
+
declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32)
+declare {i64, i1} @llvm.uadd.with.overflow.i64(i64, i64)
declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
new file mode 100644
index 00000000000000..09c0a92d596ce3
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
@@ -0,0 +1,50 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers
+; which were used as arguments in call lowering and so caused early definition of SPIR-V types.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_id = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_bf16 = type { i16 }
+
+define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) {
+entry:
+ %FloatArray.i = alloca [4 x float], align 4
+ %BF16Array.i = alloca [4 x %t_bf16], align 2
+ %0 = load i64, ptr %_arg_ERR3, align 8
+ %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0
+ %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4)
+ %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4)
+ call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i)
+ br label %for.cond.i
+
+for.cond.i: ; preds = %for.inc.i, %entry
+ %lsr.iv1 = phi ptr [ %scevgep2, %for.inc.i ], [ %FloatArray.i, %entry ]
+ %lsr.iv = phi ptr addrspace(4) [ %scevgep, %for.inc.i ], [ %BF16Array.ascast.i, %entry ]
+ %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ]
+ %cmp.i = icmp ult i32 %i.0.i, 4
+ br i1 %cmp.i, label %for.body.i, label %exit
+
+for.body.i: ; preds = %for.cond.i
+ %1 = load float, ptr %lsr.iv1, align 4
+ %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %lsr.iv)
+ %cmp5.i = fcmp une float %1, %call.i.i
+ br i1 %cmp5.i, label %if.then.i, label %for.inc.i
+
+if.then.i: ; preds = %for.body.i
+ store i32 1, ptr addrspace(1) %add.ptr.i, align 4
+ br label %for.inc.i
+
+for.inc.i: ; preds = %if.then.i, %for.body.i
+ %inc.i = add nuw nsw i32 %i.0.i, 1
+ %scevgep = getelementptr i8, ptr addrspace(4) %lsr.iv, i64 2
+ %scevgep2 = getelementptr i8, ptr %lsr.iv1, i64 4
+ br label %for.cond.i
+
+exit: ; preds = %for.cond.i
+ ret void
+}
+
+declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg)
+declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4))
+declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
new file mode 100644
index 00000000000000..a31638e0b87043
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
@@ -0,0 +1,49 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers
+; which were used as arguments in call lowering and so caused early definition of SPIR-V types.
+
+; RUN: %if spirv-tools %{ llc -O2 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_id = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_bf16 = type { i16 }
+
+define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) {
+entry:
+ %FloatArray.i = alloca [4 x float], align 4
+ %BF16Array.i = alloca [4 x %t_bf16], align 2
+ %0 = load i64, ptr %_arg_ERR3, align 8
+ %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0
+ %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4)
+ %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4)
+ call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i)
+ br label %for.cond.i
+
+for.cond.i: ; preds = %for.inc.i, %entry
+ %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ]
+ %cmp.i = icmp ult i32 %i.0.i, 4
+ br i1 %cmp.i, label %for.body.i, label %exit
+
+for.body.i: ; preds = %for.cond.i
+ %idxprom.i = zext nneg i32 %i.0.i to i64
+ %arrayidx.i = getelementptr inbounds [4 x float], ptr %FloatArray.i, i64 0, i64 %idxprom.i
+ %1 = load float, ptr %arrayidx.i, align 4
+ %arrayidx4.i = getelementptr inbounds [4 x %t_bf16], ptr addrspace(4) %BF16Array.ascast.i, i64 0, i64 %idxprom.i
+ %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %arrayidx4.i)
+ %cmp5.i = fcmp une float %1, %call.i.i
+ br i1 %cmp5.i, label %if.then.i, label %for.inc.i
+
+if.then.i: ; preds = %for.body.i
+ store i32 1, ptr addrspace(1) %add.ptr.i, align 4
+ br label %for.inc.i
+
+for.inc.i: ; preds = %if.then.i, %for.body.i
+ %inc.i = add nuw nsw i32 %i.0.i, 1
+ br label %for.cond.i
+
+exit: ; preds = %for.cond.i
+ ret void
+}
+
+declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg)
+declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4))
+declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))
More information about the llvm-commits
mailing list