[llvm] [SPIR-V] Implement builtins for OpIAddCarry/OpISubBorrow and improve/fix type inference (PR #115192)
Vyacheslav Levytskyy via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 12 03:59:31 PST 2024
https://github.com/VyacheslavLevytskyy updated https://github.com/llvm/llvm-project/pull/115192
>From 02bf2fd850e118e2c25ceed57c07ce587acd17ab Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 6 Nov 2024 09:18:01 -0800
Subject: [PATCH 1/5] call lowering may not know register's type
---
llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 98cf598a1f031a..d914f3cf256d48 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -545,13 +545,23 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
Register ArgReg = Arg.Regs[0];
ArgVRegs.push_back(ArgReg);
SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg);
- if (!SpvType) {
+ // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) we should
+ // wait with setting the type for the virtual register until pre-legalizer
+ // step when we access @llvm.spv.assign.ptr.type.p...(...)'s info.
+ if (!SpvType && !isUntypedPointerTy(Arg.Ty)) {
SpvType = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder);
GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
}
if (!MRI->getRegClassOrNull(ArgReg)) {
- MRI->setRegClass(ArgReg, GR->getRegClass(SpvType));
- MRI->setType(ArgReg, GR->getRegType(SpvType));
+ // Either we have SpvType created, or Arg.Ty is an untyped pointer and
+ // we know its virtual register's class and type.
+ MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
+ : &SPIRV::pIDRegClass);
+ MRI->setType(
+ ArgReg,
+ SpvType ? GR->getRegType(SpvType)
+ : LLT::pointer(cast<PointerType>(Arg.Ty)->getAddressSpace(),
+ GR->getPointerSize()));
}
}
auto instructionSet = canUseOpenCL ? SPIRV::InstructionSet::OpenCL_std
>From 86837dff62ef5634ba95c373579243b6943ddd63 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 6 Nov 2024 10:46:13 -0800
Subject: [PATCH 2/5] ensure that correct types are applied to virtual
registers which were used as arguments in call lowering and so caused early
definition of SPIR-V types
---
llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 27 +++++++---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 2 +
...operand-types-vs-calllowering-unwrapped.ll | 50 +++++++++++++++++++
...phi-valid-operand-types-vs-calllowering.ll | 49 ++++++++++++++++++
4 files changed, 121 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
create mode 100644 llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index d914f3cf256d48..8d1b82465d3df2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -545,16 +545,29 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
Register ArgReg = Arg.Regs[0];
ArgVRegs.push_back(ArgReg);
SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg);
- // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) we should
- // wait with setting the type for the virtual register until pre-legalizer
- // step when we access @llvm.spv.assign.ptr.type.p...(...)'s info.
- if (!SpvType && !isUntypedPointerTy(Arg.Ty)) {
- SpvType = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder);
- GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
+ if (!SpvType) {
+ Type *ArgTy = nullptr;
+ if (auto *PtrArgTy = dyn_cast<PointerType>(Arg.Ty)) {
+ // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we
+ // don't have access to original value in LLVM IR or info about
+ // deduced pointee type, then we should wait with setting the type for
+ // the virtual register until pre-legalizer step when we access
+ // @llvm.spv.assign.ptr.type.p...(...)'s info.
+ if (Arg.OrigValue)
+ if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue))
+ ArgTy = TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
+ } else {
+ ArgTy = Arg.Ty;
+ }
+ if (ArgTy) {
+ SpvType = GR->getOrCreateSPIRVType(ArgTy, MIRBuilder);
+ GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
+ }
}
if (!MRI->getRegClassOrNull(ArgReg)) {
// Either we have SpvType created, or Arg.Ty is an untyped pointer and
- // we know its virtual register's class and type.
+ // we know its virtual register's class and type even if we don't know
+ // pointee type.
MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
: &SPIRV::pIDRegClass);
MRI->setType(
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 8b7e9c48de6c75..191533a52cccd9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1219,6 +1219,8 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(AddressSpace)};
auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
I->setOperand(OperandToReplace, PtrCastI);
+ // We need to set up a pointee type for the newly created spv_ptrcast.
+ buildAssignPtr(B, ExpectedElementType, PtrCastI);
}
void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
new file mode 100644
index 00000000000000..09c0a92d596ce3
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
@@ -0,0 +1,50 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers
+; which were used as arguments in call lowering and so caused early definition of SPIR-V types.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_id = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_bf16 = type { i16 }
+
+define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) {
+entry:
+ %FloatArray.i = alloca [4 x float], align 4
+ %BF16Array.i = alloca [4 x %t_bf16], align 2
+ %0 = load i64, ptr %_arg_ERR3, align 8
+ %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0
+ %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4)
+ %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4)
+ call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i)
+ br label %for.cond.i
+
+for.cond.i: ; preds = %for.inc.i, %entry
+ %lsr.iv1 = phi ptr [ %scevgep2, %for.inc.i ], [ %FloatArray.i, %entry ]
+ %lsr.iv = phi ptr addrspace(4) [ %scevgep, %for.inc.i ], [ %BF16Array.ascast.i, %entry ]
+ %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ]
+ %cmp.i = icmp ult i32 %i.0.i, 4
+ br i1 %cmp.i, label %for.body.i, label %exit
+
+for.body.i: ; preds = %for.cond.i
+ %1 = load float, ptr %lsr.iv1, align 4
+ %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %lsr.iv)
+ %cmp5.i = fcmp une float %1, %call.i.i
+ br i1 %cmp5.i, label %if.then.i, label %for.inc.i
+
+if.then.i: ; preds = %for.body.i
+ store i32 1, ptr addrspace(1) %add.ptr.i, align 4
+ br label %for.inc.i
+
+for.inc.i: ; preds = %if.then.i, %for.body.i
+ %inc.i = add nuw nsw i32 %i.0.i, 1
+ %scevgep = getelementptr i8, ptr addrspace(4) %lsr.iv, i64 2
+ %scevgep2 = getelementptr i8, ptr %lsr.iv1, i64 4
+ br label %for.cond.i
+
+exit: ; preds = %for.cond.i
+ ret void
+}
+
+declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg)
+declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4))
+declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
new file mode 100644
index 00000000000000..a31638e0b87043
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
@@ -0,0 +1,49 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers
+; which were used as arguments in call lowering and so caused early definition of SPIR-V types.
+
+; RUN: %if spirv-tools %{ llc -O2 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_id = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_bf16 = type { i16 }
+
+define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) {
+entry:
+ %FloatArray.i = alloca [4 x float], align 4
+ %BF16Array.i = alloca [4 x %t_bf16], align 2
+ %0 = load i64, ptr %_arg_ERR3, align 8
+ %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0
+ %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4)
+ %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4)
+ call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i)
+ br label %for.cond.i
+
+for.cond.i: ; preds = %for.inc.i, %entry
+ %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ]
+ %cmp.i = icmp ult i32 %i.0.i, 4
+ br i1 %cmp.i, label %for.body.i, label %exit
+
+for.body.i: ; preds = %for.cond.i
+ %idxprom.i = zext nneg i32 %i.0.i to i64
+ %arrayidx.i = getelementptr inbounds [4 x float], ptr %FloatArray.i, i64 0, i64 %idxprom.i
+ %1 = load float, ptr %arrayidx.i, align 4
+ %arrayidx4.i = getelementptr inbounds [4 x %t_bf16], ptr addrspace(4) %BF16Array.ascast.i, i64 0, i64 %idxprom.i
+ %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %arrayidx4.i)
+ %cmp5.i = fcmp une float %1, %call.i.i
+ br i1 %cmp5.i, label %if.then.i, label %for.inc.i
+
+if.then.i: ; preds = %for.body.i
+ store i32 1, ptr addrspace(1) %add.ptr.i, align 4
+ br label %for.inc.i
+
+for.inc.i: ; preds = %if.then.i, %for.body.i
+ %inc.i = add nuw nsw i32 %i.0.i, 1
+ br label %for.cond.i
+
+exit: ; preds = %for.cond.i
+ ret void
+}
+
+declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg)
+declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4))
+declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))
>From cc22d56e9d779ebea51a083dfa64663232964f46 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Thu, 7 Nov 2024 02:53:37 -0800
Subject: [PATCH 3/5] clang-format
---
llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 8d1b82465d3df2..3c5397319aaf21 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -555,7 +555,8 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
// @llvm.spv.assign.ptr.type.p...(...)'s info.
if (Arg.OrigValue)
if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue))
- ArgTy = TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
+ ArgTy =
+ TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
} else {
ArgTy = Arg.Ty;
}
>From 07530bde7e662aba461693da8b6577d7c68bbc81 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Fri, 8 Nov 2024 03:27:04 -0800
Subject: [PATCH 4/5] implement OpIAddCarry, OpISubBorrow; improve type
inference; fix access to erased from parent after visit instructions; fix
validation of pointer types for frexp and lgamma_r; improve parsing of
builtin names
---
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 69 +++++++++-
llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 5 +
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 48 +++++--
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 18 +++
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 7 +-
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 19 +--
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 20 +++
llvm/lib/Target/SPIRV/SPIRVUtils.h | 5 +
llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll | 129 ++++++++++++++++++
llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll | 127 +++++++++++++++++
10 files changed, 410 insertions(+), 37 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
create mode 100644 llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 9aa1c31db7d208..06a37f1f559d44 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -190,8 +190,14 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) {
// Check if the extracted name contains type information between angle
// brackets. If so, the builtin is an instantiated template - needs to have
// the information after angle brackets and return type removed.
- if (BuiltinName.find('<') && BuiltinName.back() == '>') {
- BuiltinName = BuiltinName.substr(0, BuiltinName.find('<'));
+ std::size_t Pos1 = BuiltinName.rfind('<');
+ if (Pos1 != std::string::npos && BuiltinName.back() == '>') {
+ std::size_t Pos2 = BuiltinName.rfind(' ', Pos1);
+ if (Pos2 == std::string::npos)
+ Pos2 = 0;
+ else
+ ++Pos2;
+ BuiltinName = BuiltinName.substr(Pos2, Pos1 - Pos2);
BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(' ') + 1);
}
@@ -461,9 +467,11 @@ static Register buildBuiltinVariableLoad(
SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType,
Register Reg = Register(0), bool isConst = true, bool hasLinkageTy = true) {
Register NewRegister =
- MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
- MIRBuilder.getMRI()->setType(NewRegister,
- LLT::pointer(0, GR->getPointerSize()));
+ MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::pIDRegClass);
+ MIRBuilder.getMRI()->setType(
+ NewRegister,
+ LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
+ GR->getPointerSize()));
SPIRVType *PtrType = GR->getOrCreateSPIRVPointerType(
VariableType, MIRBuilder, SPIRV::StorageClass::Input);
GR->assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
@@ -1556,6 +1564,55 @@ static bool generateWaveInst(const SPIRV::IncomingCall *Call,
/* isConst= */ false, /* hasLinkageTy= */ false);
}
+// We expect a builtin
+// Name(ptr sret([RetType]) %result, Type %operand1, Type %operand1)
+// where %result is a pointer to where the result of the builtin execution
+// is to be stored, and generate the following instructions:
+// Res = Opcode RetType Operand1 Operand1
+// OpStore RetVariable Res
+static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+ unsigned Opcode =
+ SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+
+ Register SRetReg = Call->Arguments[0];
+ SPIRVType *PtrRetType = GR->getSPIRVTypeForVReg(SRetReg);
+ SPIRVType *RetType = GR->getPointeeType(PtrRetType);
+ if (!RetType)
+ report_fatal_error("The first parameter must be a pointer");
+ if (RetType->getOpcode() != SPIRV::OpTypeStruct)
+ report_fatal_error("Expected struct type result for the arithmetic with "
+ "overflow builtins");
+
+ SPIRVType *OpType1 = GR->getSPIRVTypeForVReg(Call->Arguments[1]);
+ SPIRVType *OpType2 = GR->getSPIRVTypeForVReg(Call->Arguments[2]);
+ if (!OpType1 || !OpType2 || OpType1 != OpType2)
+ report_fatal_error("Operands must have the same type");
+ if (OpType1->getOpcode() == SPIRV::OpTypeVector)
+ switch (Opcode) {
+ case SPIRV::OpIAddCarryS:
+ Opcode = SPIRV::OpIAddCarryV;
+ break;
+ case SPIRV::OpISubBorrowS:
+ Opcode = SPIRV::OpISubBorrowV;
+ break;
+ }
+
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ Register ResReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+ MRI->setRegClass(ResReg, &SPIRV::iIDRegClass);
+ GR->assignSPIRVTypeToVReg(RetType, ResReg, MIRBuilder.getMF());
+ MIRBuilder.buildInstr(Opcode)
+ .addDef(ResReg)
+ .addUse(GR->getSPIRVTypeID(RetType))
+ .addUse(Call->Arguments[1])
+ .addUse(Call->Arguments[2]);
+ MIRBuilder.buildInstr(SPIRV::OpStore).addUse(SRetReg).addUse(ResReg);
+ return true;
+}
+
static bool generateGetQueryInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
@@ -2511,6 +2568,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
return generateDotOrFMulInst(Call.get(), MIRBuilder, GR);
case SPIRV::Wave:
return generateWaveInst(Call.get(), MIRBuilder, GR);
+ case SPIRV::ICarryBorrow:
+ return generateICarryBorrowInst(Call.get(), MIRBuilder, GR);
case SPIRV::GetQuery:
return generateGetQueryInst(Call.get(), MIRBuilder, GR);
case SPIRV::ImageSizeQuery:
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 29f11c3dd36865..1b95b1479bb932 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -63,6 +63,7 @@ def KernelClock : BuiltinGroup;
def CastToPtr : BuiltinGroup;
def Construct : BuiltinGroup;
def CoopMatr : BuiltinGroup;
+def ICarryBorrow : BuiltinGroup;
//===----------------------------------------------------------------------===//
// Class defining a demangled builtin record. The information in the record
@@ -628,6 +629,10 @@ defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBar
defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>;
+// ICarryBorrow builtin record:
+defm : DemangledNativeBuiltin<"__spirv_IAddCarry", OpenCL_std, ICarryBorrow, 3, 3, OpIAddCarryS>;
+defm : DemangledNativeBuiltin<"__spirv_ISubBorrow", OpenCL_std, ICarryBorrow, 3, 3, OpISubBorrowS>;
+
// cl_intel_split_work_group_barrier
defm : DemangledNativeBuiltin<"intel_work_group_barrier_arrive", OpenCL_std, Barrier, 1, 2, OpControlBarrierArriveINTEL>;
defm : DemangledNativeBuiltin<"__spirv_ControlBarrierArriveINTEL", OpenCL_std, Barrier, 3, 3, OpControlBarrierArriveINTEL>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 191533a52cccd9..e6ef40e010dc20 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -76,8 +76,8 @@ class SPIRVEmitIntrinsics
SPIRV::InstructionSet::InstructionSet InstrSet;
// a register of Instructions that don't have a complete type definition
- SmallPtrSet<Value *, 8> UncompleteTypeInfo;
- SmallVector<Instruction *> PostprocessWorklist;
+ DenseMap<Value *, unsigned> UncompleteTypeInfo;
+ SmallVector<Value *> PostprocessWorklist;
// well known result types of builtins
enum WellKnownTypes { Event };
@@ -147,6 +147,7 @@ class SPIRVEmitIntrinsics
std::unordered_set<Function *> &FVisited);
void replaceWithPtrcasted(Instruction *CI, Type *NewElemTy, Type *KnownElemTy,
CallInst *AssignCI);
+ void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
bool runOnFunction(Function &F);
bool postprocessTypes();
@@ -272,6 +273,27 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
false);
}
+void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
+ bool DeleteOld) {
+ Src->replaceAllUsesWith(Dest);
+ // Update deduced type records
+ GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld);
+ GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld);
+ // Update uncomplete type records if any
+ auto It = UncompleteTypeInfo.find(Src);
+ if (It == UncompleteTypeInfo.end())
+ return;
+ if (DeleteOld) {
+ unsigned Pos = It->second;
+ UncompleteTypeInfo.erase(Src);
+ UncompleteTypeInfo[Dest] = Pos;
+ PostprocessWorklist[Pos] = Dest;
+ } else {
+ UncompleteTypeInfo[Dest] = PostprocessWorklist.size();
+ PostprocessWorklist.push_back(Dest);
+ }
+}
+
static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
isPointerTy(SI->getValueOperand()->getType()) &&
@@ -434,7 +456,7 @@ void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
if (!UnknownElemTypeI8)
return;
if (auto *I = dyn_cast<Instruction>(Op)) {
- UncompleteTypeInfo.insert(I);
+ UncompleteTypeInfo[I] = PostprocessWorklist.size();
PostprocessWorklist.push_back(I);
}
}
@@ -640,7 +662,7 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
if (!UnknownElemTypeI8)
return nullptr;
if (auto *Instr = dyn_cast<Instruction>(I)) {
- UncompleteTypeInfo.insert(Instr);
+ UncompleteTypeInfo[Instr] = PostprocessWorklist.size();
PostprocessWorklist.push_back(Instr);
}
return IntegerType::getInt8Ty(I->getContext());
@@ -1062,7 +1084,7 @@ Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
{I.getOperand(0)->getType()}, {Args});
// remove switch to avoid its unneeded and undesirable unwrap into branches
// and conditions
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
// insert artificial and temporary instruction to preserve valid CFG,
// it will be removed after IR translation pass
@@ -1084,7 +1106,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
for (auto &Op : I.operands())
Args.push_back(Op);
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
return NewI;
}
@@ -1099,7 +1121,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
// such bitcasts do not provide sufficient information, should be just skipped
// here, and handled in insertPtrCastOrAssignTypeInstr.
if (isPointerTy(I.getType())) {
- I.replaceAllUsesWith(Source);
+ replaceAllUsesWith(&I, Source);
I.eraseFromParent();
return nullptr;
}
@@ -1108,7 +1130,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
SmallVector<Value *> Args(I.op_begin(), I.op_end());
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1333,7 +1355,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
SmallVector<Value *> Args(I.op_begin(), I.op_end());
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1348,7 +1370,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1384,7 +1406,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
Args.push_back(B.getInt32(Op));
auto *NewI =
B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
return NewI;
}
@@ -1445,7 +1467,7 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
{PtrTy, ArraySize->getType()}, {ArraySize})
: B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
std::string InstName = I.hasName() ? I.getName().str() : "";
- I.replaceAllUsesWith(NewI);
+ replaceAllUsesWith(&I, NewI);
I.eraseFromParent();
NewI->setName(InstName);
return NewI;
@@ -1615,7 +1637,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
auto *NewOp =
buildIntrWithMD(Intrinsic::spv_track_constant,
{II->getType(), II->getType()}, t->second, I, {}, B);
- I->replaceAllUsesWith(NewOp);
+ replaceAllUsesWith(I, NewOp, false);
NewOp->setArgOperand(0, I);
}
bool IsPhi = isa<PHINode>(I), BPrepared = false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index a95b488960c4c3..2805300ae4437d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -180,6 +180,15 @@ class SPIRVGlobalRegistry {
auto It = AssignPtrTypeInstr.find(Val);
return It == AssignPtrTypeInstr.end() ? nullptr : It->second;
}
+ // - Find a record and update its key or add a new record, if found.
+ void updateIfExistAssignPtrTypeInstr(Value *OldVal, Value *NewVal,
+ bool DeleteOld) {
+ if (CallInst *CI = findAssignPtrTypeInstr(OldVal)) {
+ if (DeleteOld)
+ AssignPtrTypeInstr.erase(OldVal);
+ AssignPtrTypeInstr[NewVal] = CI;
+ }
+ }
// A registry of mutated values
// (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`):
@@ -214,6 +223,15 @@ class SPIRVGlobalRegistry {
auto It = DeducedElTys.find(Val);
return It == DeducedElTys.end() ? nullptr : It->second;
}
+ // - Find a record and update its key or add a new record, if found.
+ void updateIfExistDeducedElementType(Value *OldVal, Value *NewVal,
+ bool DeleteOld) {
+ if (Type *Ty = findDeducedElementType(OldVal)) {
+ if (DeleteOld)
+ DeducedElTys.erase(OldVal);
+ DeducedElTys[NewVal] = Ty;
+ }
+ }
// - Add a record to the map of deduced composite types.
void addDeducedCompositeType(Value *Val, Type *Ty) {
DeducedNestedTys[Val] = Ty;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index ecbceb5b472fa1..d9a79d7e525b4b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -473,8 +473,11 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)
continue;
switch (MI.getOperand(3).getImm()) {
+ case SPIRV::OpenCLExtInst::frexp:
+ case SPIRV::OpenCLExtInst::lgamma_r:
case SPIRV::OpenCLExtInst::remquo: {
- // The last operand must be of a pointer to the return type.
+ // The last operand must be of a pointer to i32 or vector of i32
+ // values.
MachineIRBuilder MIB(MI);
SPIRVType *Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB);
SPIRVType *RetType = MRI->getVRegDef(MI.getOperand(1).getReg());
@@ -487,8 +490,6 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
Int32Type, RetType->getOperand(2).getImm(), MIB));
} break;
case SPIRV::OpenCLExtInst::fract:
- case SPIRV::OpenCLExtInst::frexp:
- case SPIRV::OpenCLExtInst::lgamma_r:
case SPIRV::OpenCLExtInst::modf:
case SPIRV::OpenCLExtInst::sincos:
// The last operand must be of a pointer to the base type represented
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index a968b65c173067..f9bfb80d6eede2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -3051,22 +3051,9 @@ bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
MachineInstr &I) const {
// Change order of instructions if needed: all OpVariable instructions in a
// function must be the first instructions in the first block
- MachineFunction *MF = I.getParent()->getParent();
- MachineBasicBlock *MBB = &MF->front();
- auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end();
- bool IsHeader = false;
- unsigned Opcode;
- for (; It != E && It != I; ++It) {
- Opcode = It->getOpcode();
- if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
- IsHeader = true;
- } else if (IsHeader &&
- !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
- ++It;
- break;
- }
- }
- return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
+ auto It = getOpVariableMBBIt(I);
+ return BuildMI(*It->getParent(), It, It->getDebugLoc(),
+ TII.get(SPIRV::OpVariable))
.addDef(ResVReg)
.addUse(GR.getSPIRVTypeID(ResType))
.addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index f9b361e163c909..aeb2c29f7b8618 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -162,6 +162,26 @@ void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
}
}
+MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I) {
+ MachineFunction *MF = I.getParent()->getParent();
+ MachineBasicBlock *MBB = &MF->front();
+ MachineBasicBlock::iterator It = MBB->SkipPHIsAndLabels(MBB->begin()),
+ E = MBB->end();
+ bool IsHeader = false;
+ unsigned Opcode;
+ for (; It != E && It != I; ++It) {
+ Opcode = It->getOpcode();
+ if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
+ IsHeader = true;
+ } else if (IsHeader &&
+ !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
+ ++It;
+ break;
+ }
+ }
+ return It;
+}
+
SPIRV::StorageClass::StorageClass
addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
switch (AddrSpace) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 11fd3a5c61dcae..298b0b93b0e4d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -15,6 +15,7 @@
#include "MCTargetDesc/SPIRVBaseInfo.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/TypedPointerType.h"
@@ -139,6 +140,10 @@ void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
const MDNode *GVarMD);
+// Return a valid position for the OpVariable instruction inside a function,
+// i.e., at the beginning of the first block of the function.
+MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I);
+
// Convert a SPIR-V storage class to the corresponding LLVM IR address space.
// TODO: maybe the following two functions should be handled in the subtarget
// to allow for different OpenCL vs Vulkan handling.
diff --git a/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
new file mode 100644
index 00000000000000..8f14eba21b63a2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
@@ -0,0 +1,129 @@
+; Adapted from Khronos Translator test suite: test/iaddcarry_builtin.ll
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%i8struct = type {i8, i8}
+%i16struct = type {i16, i16}
+%i32struct = type {i32, i32}
+%i64struct = type {i64, i64}
+%vecstruct = type {<4 x i32>, <4 x i32>}
+
+; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
+; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
+; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32
+; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
+; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
+; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
+; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
+; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
+; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
+; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
+; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
+; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
+
+define spir_func void @test_builtin_iaddcarrycc(i8 %a, i8 %b) {
+ entry:
+ %0 = alloca %i8struct
+ call void @_Z17__spirv_IAddCarrycc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
+; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpIAddCarry [[i8struct]] [[a]] [[b]]
+; CHECK-SPIRV: OpStore [[var_11]] [[var_12]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) {
+ entry:
+ %0 = alloca %i16struct
+ call void @_Z17__spirv_IAddCarryss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
+; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpIAddCarry [[i16struct]] [[a_0]] [[b_0]]
+; CHECK-SPIRV: OpStore [[var_21]] [[var_22]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %i32struct
+ call void @_Z17__spirv_IAddCarryii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
+; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpIAddCarry [[i32struct]] [[a_1]] [[b_1]]
+; CHECK-SPIRV: OpStore [[var_31]] [[var_32]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) {
+ entry:
+ %0 = alloca %i64struct
+ call void @_Z17__spirv_IAddCarryll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
+; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpIAddCarry [[i64struct]] [[a_2]] [[b_2]]
+; CHECK-SPIRV: OpStore [[var_41]] [[var_42]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
+ entry:
+ %0 = alloca %vecstruct
+ call void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
+; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpIAddCarry [[vecstruct]] [[a_3]] [[b_3]]
+; CHECK-SPIRV: OpStore [[var_51]] [[var_52]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+%struct.anon = type { i32, i32 }
+
+define spir_func void @test_builtin_iaddcarry_anon(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %struct.anon
+ %1 = addrspacecast ptr %0 to ptr addrspace(4)
+ call spir_func void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
+; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
+; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpIAddCarry [[struct_anon]] [[a_4]] [[b_4]]
+; CHECK-SPIRV: OpStore [[var_61]] [[var_62]]
+
+declare void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
+declare void @_Z17__spirv_IAddCarrycc(ptr sret(%i8struct), i8, i8)
+declare void @_Z17__spirv_IAddCarryss(ptr sret(%i16struct), i16, i16)
+declare void @_Z17__spirv_IAddCarryii(ptr sret(%i32struct), i32, i32)
+declare void @_Z17__spirv_IAddCarryll(ptr sret(%i64struct), i64, i64)
+declare void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)
diff --git a/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
new file mode 100644
index 00000000000000..08b4d2a1fa8e55
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
@@ -0,0 +1,127 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%i8struct = type {i8, i8}
+%i16struct = type {i16, i16}
+%i32struct = type {i32, i32}
+%i64struct = type {i64, i64}
+%vecstruct = type {<4 x i32>, <4 x i32>}
+
+; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
+; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
+; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32
+; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
+; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
+; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
+; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
+; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
+; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
+; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
+; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
+; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
+
+define spir_func void @test_builtin_isubborrowcc(i8 %a, i8 %b) {
+ entry:
+ %0 = alloca %i8struct
+ call void @_Z18__spirv_ISubBorrowcc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
+; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpISubBorrow [[i8struct]] [[a]] [[b]]
+; CHECK-SPIRV: OpStore [[var_11]] [[var_12]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowss(i16 %a, i16 %b) {
+ entry:
+ %0 = alloca %i16struct
+ call void @_Z18__spirv_ISubBorrowss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
+; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpISubBorrow [[i16struct]] [[a_0]] [[b_0]]
+; CHECK-SPIRV: OpStore [[var_21]] [[var_22]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowii(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %i32struct
+ call void @_Z18__spirv_ISubBorrowii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
+; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpISubBorrow [[i32struct]] [[a_1]] [[b_1]]
+; CHECK-SPIRV: OpStore [[var_31]] [[var_32]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowll(i64 %a, i64 %b) {
+ entry:
+ %0 = alloca %i64struct
+ call void @_Z18__spirv_ISubBorrowll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
+; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpISubBorrow [[i64struct]] [[a_2]] [[b_2]]
+; CHECK-SPIRV: OpStore [[var_41]] [[var_42]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
+ entry:
+ %0 = alloca %vecstruct
+ call void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
+; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpISubBorrow [[vecstruct]] [[a_3]] [[b_3]]
+; CHECK-SPIRV: OpStore [[var_51]] [[var_52]]
+; CHECK-SPIRV: OpReturn
+; CHECK-SPIRV: OpFunctionEnd
+
+%struct.anon = type { i32, i32 }
+
+define spir_func void @test_builtin_isubborrow_anon(i32 %a, i32 %b) {
+ entry:
+ %0 = alloca %struct.anon
+ %1 = addrspacecast ptr %0 to ptr addrspace(4)
+ call spir_func void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
+ ret void
+}
+; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
+; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
+; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpISubBorrow [[struct_anon]] [[a_4]] [[b_4]]
+; CHECK-SPIRV: OpStore [[var_61]] [[var_62]]
+
+declare void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
+declare void @_Z18__spirv_ISubBorrowcc(ptr sret(%i8struct), i8, i8)
+declare void @_Z18__spirv_ISubBorrowss(ptr sret(%i16struct), i16, i16)
+declare void @_Z18__spirv_ISubBorrowii(ptr sret(%i32struct), i32, i32)
+declare void @_Z18__spirv_ISubBorrowll(ptr sret(%i64struct), i64, i64)
+declare void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)
>From 980f2b6ad0a92d98cc52325d7434da7a8ab58137 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 12 Nov 2024 03:19:14 -0800
Subject: [PATCH 5/5] LLVM IR add/sub instructions result in logical SPIR-V
instructions when applied to bool type
---
llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 11 ++++++
.../instructions/scalar-integer-arithmetic.ll | 34 +++++++++++++++++++
2 files changed, 45 insertions(+)
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index d9a79d7e525b4b..59a1bf50b771b9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -414,6 +414,17 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
validateForwardCalls(STI, MRI, GR, MI);
break;
+ // ensure that LLVM IR add/sub instructions result in logical SPIR-V
+ // instructions when applied to bool type
+ case SPIRV::OpIAddS:
+ case SPIRV::OpIAddV:
+ case SPIRV::OpISubS:
+ case SPIRV::OpISubV:
+ if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
+ SPIRV::OpTypeBool))
+ MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
+ break;
+
// ensure that LLVM IR bitwise instructions result in logical SPIR-V
// instructions when applied to bool type
case SPIRV::OpBitwiseOrS:
diff --git a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
index d222dfa570cf78..411866a67cc67b 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
@@ -1,5 +1,11 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; CHECK-DAG: OpName [[BOOL_ADD:%.+]] "bool_add"
+; CHECK-DAG: OpName [[BOOL_SUB:%.+]] "bool_sub"
; CHECK-DAG: OpName [[SCALAR_ADD:%.+]] "scalar_add"
; CHECK-DAG: OpName [[SCALAR_SUB:%.+]] "scalar_sub"
; CHECK-DAG: OpName [[SCALAR_MUL:%.+]] "scalar_mul"
@@ -10,13 +16,28 @@
; CHECK-NOT: DAG-FENCE
+; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
; CHECK-DAG: [[SCALAR:%.+]] = OpTypeInt 32
; CHECK-DAG: [[SCALAR_FN:%.+]] = OpTypeFunction [[SCALAR]] [[SCALAR]] [[SCALAR]]
+; CHECK-DAG: [[BOOL_FN:%.+]] = OpTypeFunction [[BOOL]] [[BOOL]] [[BOOL]]
; CHECK-NOT: DAG-FENCE
;; Test add on scalar:
+define i1 @bool_add(i1 %a, i1 %b) {
+ %c = add i1 %a, %b
+ ret i1 %c
+}
+
+; CHECK: [[BOOL_ADD]] = OpFunction [[BOOL]] None [[BOOL_FN]]
+; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK: OpLabel
+; CHECK: [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]]
+; CHECK: OpReturnValue [[C]]
+; CHECK-NEXT: OpFunctionEnd
+
define i32 @scalar_add(i32 %a, i32 %b) {
%c = add i32 %a, %b
ret i32 %c
@@ -32,6 +53,19 @@ define i32 @scalar_add(i32 %a, i32 %b) {
;; Test sub on scalar:
+define i1 @bool_sub(i1 %a, i1 %b) {
+ %c = sub i1 %a, %b
+ ret i1 %c
+}
+
+; CHECK: [[BOOL_SUB]] = OpFunction [[BOOL]] None [[BOOL_FN]]
+; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK: OpLabel
+; CHECK: [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]]
+; CHECK: OpReturnValue [[C]]
+; CHECK-NEXT: OpFunctionEnd
+
define i32 @scalar_sub(i32 %a, i32 %b) {
%c = sub i32 %a, %b
ret i32 %c
More information about the llvm-commits
mailing list