[llvm] [SPIR-V] Allow intrinsics with aggregate return type to reach GlobalISel (PR #108893)
Vyacheslav Levytskyy via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 19 02:56:22 PDT 2024
https://github.com/VyacheslavLevytskyy updated https://github.com/llvm/llvm-project/pull/108893
>From 91c96690c0c2ecf30707e9179db3becba077cd7b Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Mon, 16 Sep 2024 15:00:55 -0700
Subject: [PATCH 01/12] allow intrinsics with aggregate return type to reach
GlobalISel
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 48 ++++++++++++++++++-
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 3 +-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 8 +++-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 19 ++++++++
4 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 4175f766ac69ad..0af656729c946f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -188,6 +188,40 @@ bool isConvergenceIntrinsic(const Instruction *I) {
II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
}
+
+bool isInternalNonVoidIntrinsic(const Value *I) {
+ if (const auto *II = dyn_cast<IntrinsicInst>(I))
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::spv_cmpxchg:
+ case Intrinsic::spv_const_composite:
+ case Intrinsic::spv_track_constant:
+ case Intrinsic::spv_load:
+ case Intrinsic::spv_extractv:
+ case Intrinsic::spv_insertv:
+ case Intrinsic::spv_extractelt:
+ case Intrinsic::spv_insertelt:
+ case Intrinsic::spv_bitcast:
+ case Intrinsic::spv_ptrcast:
+ case Intrinsic::spv_alloca:
+ case Intrinsic::spv_alloca_array:
+ case Intrinsic::spv_undef:
+ return true;
+ }
+ return false;
+}
+
+bool allowEmitFakeUse(const Value *Arg) {
+ if (isInternalNonVoidIntrinsic(Arg))
+ return false;
+ if (dyn_cast<AtomicCmpXchgInst>(Arg) || dyn_cast<InsertValueInst>(Arg) ||
+ dyn_cast<UndefValue>(Arg))
+ return false;
+ if (const auto *LI = dyn_cast<LoadInst>(Arg))
+ if (LI->getType()->isAggregateType())
+ return false;
+ return true;
+}
+
} // namespace
char SPIRVEmitIntrinsics::ID = 0;
@@ -283,8 +317,16 @@ static inline Type *reconstructType(SPIRVGlobalRegistry *GR, Value *Op) {
void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
Value *Arg) {
Value *OfType = PoisonValue::get(Ty);
- CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
- {Arg->getType()}, OfType, Arg, {}, B);
+ CallInst *AssignCI = nullptr;
+ if (Ty->isAggregateType() && allowEmitFakeUse(Arg)) {
+ AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg});
+ AssignCI->setMetadata(
+ "spirv.__BE.assign_type",
+ MDNode::get(Arg->getContext(), ValueAsMetadata::getConstant(OfType)));
+ } else {
+ AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},
+ OfType, Arg, {}, B);
+ }
GR->addAssignPtrTypeInstr(Arg, AssignCI);
}
@@ -1270,6 +1312,8 @@ Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
}
Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
+ if (I.getAggregateOperand()->getType()->isAggregateType())
+ return &I;
IRBuilder<> B(I.getParent());
B.SetInsertPoint(&I);
SmallVector<Value *> Args;
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 9fe4d8a16bc32a..189b84369d1c33 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -287,7 +287,8 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
// TODO: add proper legalization rules.
getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal();
- getActionDefinitionsBuilder({G_UADDO, G_USUBO, G_SMULO, G_UMULO})
+ getActionDefinitionsBuilder(
+ {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_SMULO, G_UMULO})
.alwaysLegal();
// FP conversions.
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index df1b75bc1cb9eb..53a266722da21d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -376,7 +376,13 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,
.addUse(NewReg)
.addUse(GR->getSPIRVTypeID(SpvType))
.setMIFlags(Flags);
- Def->getOperand(0).setReg(NewReg);
+ for (unsigned I = 0, E = Def->getNumExplicitDefs(); I != E; ++I) {
+ MachineOperand &MO = Def->getOperand(I);
+ if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) {
+ MO.setReg(NewReg);
+ break;
+ }
+ }
return NewReg;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 29b8f8fac98e82..e755af2ff20158 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -478,6 +478,24 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
// noted in 'spv.cloned_funcs' metadata for later restoration.
Function *
SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
+ if (F->isIntrinsic())
+ // Allow intrinsics with aggregate return type to reach GlobalISel
+ switch (F->getIntrinsicID()) {
+ // Standard C/C++ Library Intrinsics
+ case Intrinsic::frexp:
+ // Vector Reduction Intrinsics
+ case Intrinsic::vector_interleave2:
+ case Intrinsic::vector_deinterleave2:
+ // Arithmetic with Overflow Intrinsics
+ case Intrinsic::smul_with_overflow:
+ case Intrinsic::umul_with_overflow:
+ case Intrinsic::sadd_with_overflow:
+ case Intrinsic::uadd_with_overflow:
+ case Intrinsic::ssub_with_overflow:
+ case Intrinsic::usub_with_overflow:
+ return F;
+ }
+
IRBuilder<> B(F->getContext());
bool IsRetAggr = F->getReturnType()->isAggregateType();
@@ -561,6 +579,7 @@ bool SPIRVPrepareFunctions::runOnModule(Module &M) {
Changed = true;
}
}
+
return Changed;
}
>From 171be7b44a168238f916c3f50df3fa7dd010c801 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 04:58:06 -0700
Subject: [PATCH 02/12] implement keeping and referencing aggregate result
attributes between passes; implement selection of Arithmetic with Overflow
Intrinsics
---
llvm/include/llvm/IR/IntrinsicsSPIRV.td | 1 +
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 11 +--
llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 17 +++++
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 70 +++++++++++++++++++
llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 22 +++++-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 17 +++--
7 files changed, 126 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index a4c01952927175..e3a875ae0b1b3e 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -15,6 +15,7 @@ let TargetPrefix = "spv" in {
def int_spv_assign_ptr_type : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
def int_spv_assign_name : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>;
def int_spv_assign_decoration : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty]>;
+ def int_spv_value_md : Intrinsic<[], [llvm_metadata_ty]>;
def int_spv_track_constant : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty]>;
def int_spv_init_global : Intrinsic<[], [llvm_any_ty, llvm_any_ty]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 0af656729c946f..e4fcfd21d5e691 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -319,10 +319,13 @@ void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
Value *OfType = PoisonValue::get(Ty);
CallInst *AssignCI = nullptr;
if (Ty->isAggregateType() && allowEmitFakeUse(Arg)) {
+ LLVMContext &Ctx = Arg->getContext();
+ SmallVector<Metadata *, 2> ArgMDs{
+ MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)),
+ MDString::get(Ctx, Arg->getName())};
+ B.CreateIntrinsic(Intrinsic::spv_value_md, {},
+ {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))});
AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg});
- AssignCI->setMetadata(
- "spirv.__BE.assign_type",
- MDNode::get(Arg->getContext(), ValueAsMetadata::getConstant(OfType)));
} else {
AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},
OfType, Arg, {}, B);
@@ -1579,7 +1582,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
I->setOperand(OpNo, NewOp);
}
}
- if (I->hasName()) {
+ if (I->hasName() && !I->getType()->isAggregateType()) {
reportFatalOnTokenType(I);
setInsertPointAfterDef(B, I);
std::vector<Value *> Args = {I};
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index a5cb86f4f1c638..1637fa8b84f132 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -55,6 +55,8 @@ class SPIRVGlobalRegistry {
// created during substitution of aggregate arguments
// (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`)
DenseMap<Value *, Type *> MutatedAggRet;
+ // map an instruction to its value's attributes (type, name)
+ DenseMap<MachineInstr *, std::pair<Type *, StringRef>> ValueAttrs;
// Look for an equivalent of the newType in the map. Return the equivalent
// if it's found, otherwise insert newType to the map and return the type.
@@ -177,6 +179,21 @@ class SPIRVGlobalRegistry {
return It == MutatedAggRet.end() ? nullptr : It->second;
}
+ // A registry of value's attributes (type, name)
+ // - Add a record.
+ void addValueAttrs(MachineInstr *Key, std::pair<Type *, StringRef> Val) {
+ ValueAttrs[Key] = Val;
+ }
+ // - Find a record.
+ bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name) {
+ auto It = ValueAttrs.find(Key);
+ if (It == ValueAttrs.end())
+ return false;
+ Ty = It->second.first;
+ Name = It->second.second;
+ return true;
+ }
+
// Deduced element types of untyped pointers and composites:
// - Add a record to the map of deduced element types.
void addDeducedElementType(Value *Val, Type *Ty) { DeducedElTys[Val] = Ty; }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index b526c9f29f1e6a..2e1dd1eeb6d729 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -191,6 +191,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectFloatDot(Register ResVReg, const SPIRVType *ResType,
MachineInstr &I) const;
+ bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType,
+ MachineInstr &I, unsigned Opcode) const;
+
bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType,
MachineInstr &I) const;
@@ -590,6 +593,23 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
case TargetOpcode::G_USUBSAT:
return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
+ case TargetOpcode::G_SADDO:
+ report_fatal_error(
+ "Selection of a signed addition with overflow is not implemented",
+ false);
+ case TargetOpcode::G_UADDO:
+ return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpIAddCarry);
+ case TargetOpcode::G_SSUBO:
+ report_fatal_error(
+ "Selection of a signed subtraction with overflow is not implemented",
+ false);
+ case TargetOpcode::G_USUBO:
+ return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpISubBorrow);
+ case TargetOpcode::G_UMULO:
+ return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
+ case TargetOpcode::G_SMULO:
+ return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended);
+
case TargetOpcode::G_SEXT:
return selectExt(ResVReg, ResType, I, true);
case TargetOpcode::G_ANYEXT:
@@ -1101,6 +1121,53 @@ bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
.constrainAllUses(TII, TRI, RBI);
}
+bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
+ const SPIRVType *ResType,
+ MachineInstr &I,
+ unsigned Opcode) const {
+ Type *ResTy = nullptr;
+ StringRef ResName;
+ if (!GR.findValueAttrs(&I, ResTy, ResName))
+ report_fatal_error(
+ "Not enough info to select the arithmetic with overflow instruction");
+ if (!ResTy->isStructTy())
+ report_fatal_error("Expect struct type result for the arithmetic "
+ "with overflow instruction");
+ // "Result Type must be from OpTypeStruct. The struct must have two members,
+ // and the two members must be the same type."
+ Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
+ ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy});
+ // Build SPIR-V type if needed.
+ MachineIRBuilder MIRBuilder(I);
+ SPIRVType *StructType = GR.getOrCreateSPIRVType(
+ ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
+ // A new virtual register to store the result struct.
+ Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+ MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
+ // Build the result name of needed.
+ if (ResName.size() > 0)
+ buildOpName(StructVReg, ResName, MIRBuilder);
+ // Build the arithmetic with overflow instruction.
+ MachineBasicBlock &BB = *I.getParent();
+ auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
+ .addDef(StructVReg)
+ .addUse(GR.getSPIRVTypeID(StructType));
+ for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
+ MIB.addUse(I.getOperand(i).getReg());
+ bool Status = MIB.constrainAllUses(TII, TRI, RBI);
+ // Build instructions to extract fields of the instruction's result.
+ for (unsigned i = 0; i < I.getNumDefs(); ++i) {
+ auto MIB =
+ BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
+ .addDef(I.getOperand(i).getReg())
+ .addUse(GR.getSPIRVTypeID(ResType))
+ .addUse(StructVReg)
+ .addImm(i);
+ Status &= MIB.constrainAllUses(TII, TRI, RBI);
+ }
+ return Status;
+}
+
bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
const SPIRVType *ResType,
MachineInstr &I) const {
@@ -2492,6 +2559,9 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
}
case Intrinsic::spv_step:
return selectStep(ResVReg, ResType, I);
+ case Intrinsic::spv_value_md:
+ // ignore the intrinsic
+ break;
default: {
std::string DiagMsg;
raw_string_ostream OS(DiagMsg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 189b84369d1c33..de9c495d4cbacc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -288,7 +288,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal();
getActionDefinitionsBuilder(
- {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_SMULO, G_UMULO})
+ {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_UMULO, G_SMULO})
.alwaysLegal();
// FP conversions.
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 53a266722da21d..49830745ddd996 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -376,9 +376,9 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,
.addUse(NewReg)
.addUse(GR->getSPIRVTypeID(SpvType))
.setMIFlags(Flags);
- for (unsigned I = 0, E = Def->getNumExplicitDefs(); I != E; ++I) {
+ for (unsigned I = 0, E = Def->getNumDefs(); I != E; ++I) {
MachineOperand &MO = Def->getOperand(I);
- if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) {
+ if (MO.getReg() == Reg) {
MO.setReg(NewReg);
break;
}
@@ -468,6 +468,24 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
Def->getOpcode() != SPIRV::ASSIGN_TYPE)
insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());
ToErase.push_back(&MI);
+ } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
+ MachineInstr *MdMI = MI.getPrevNode();
+ if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) {
+ // It's an internal service info from before IRTranslator passes.
+ MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
+ for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
+ if (MRI.getVRegDef(MI.getOperand(I).getReg()) != Def)
+ Def = nullptr;
+ if (Def) {
+ const MDNode *MD = MdMI->getOperand(1).getMetadata();
+ Type *ValueTy = cast<ValueAsMetadata>(MD->getOperand(0))->getType();
+ StringRef ValueName =
+ cast<MDString>(MD->getOperand(1))->getString();
+ GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName));
+ }
+ }
+ ToErase.push_back(&MI);
+ ToErase.push_back(MdMI);
} else if (MIOp == TargetOpcode::G_CONSTANT ||
MIOp == TargetOpcode::G_FCONSTANT ||
MIOp == TargetOpcode::G_BUILD_VECTOR) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index e755af2ff20158..78371485e0c669 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -444,10 +444,10 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
lowerFunnelShifts(II);
Changed = true;
break;
- case Intrinsic::umul_with_overflow:
- lowerUMulWithOverflow(II);
- Changed = true;
- break;
+// case Intrinsic::umul_with_overflow:
+// lowerUMulWithOverflow(II);
+// Changed = true;
+// break;
case Intrinsic::assume:
case Intrinsic::expect: {
const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(*F);
@@ -478,7 +478,11 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
// noted in 'spv.cloned_funcs' metadata for later restoration.
Function *
SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
- if (F->isIntrinsic())
+ bool IsRetAggr = F->getReturnType()->isAggregateType();
+ // Allow intrinsics with aggregate return type to reach GlobalISel
+ if (F->isIntrinsic() && IsRetAggr)
+ return F;
+ /*
// Allow intrinsics with aggregate return type to reach GlobalISel
switch (F->getIntrinsicID()) {
// Standard C/C++ Library Intrinsics
@@ -495,10 +499,10 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
case Intrinsic::usub_with_overflow:
return F;
}
+*/
IRBuilder<> B(F->getContext());
- bool IsRetAggr = F->getReturnType()->isAggregateType();
bool HasAggrArg =
std::any_of(F->arg_begin(), F->arg_end(), [](Argument &Arg) {
return Arg.getType()->isAggregateType();
@@ -579,7 +583,6 @@ bool SPIRVPrepareFunctions::runOnModule(Module &M) {
Changed = true;
}
}
-
return Changed;
}
>From c1050bf2fd3a6b852e19fbb8691890c2d894cc59 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 05:36:01 -0700
Subject: [PATCH 03/12] fix passing info and access to metadata
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 3 ++-
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 11 ++++++++---
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index e4fcfd21d5e691..9c292e253650f4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -318,7 +318,8 @@ void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
Value *Arg) {
Value *OfType = PoisonValue::get(Ty);
CallInst *AssignCI = nullptr;
- if (Ty->isAggregateType() && allowEmitFakeUse(Arg)) {
+ if (Arg->getType()->isAggregateType() && Ty->isAggregateType() &&
+ allowEmitFakeUse(Arg)) {
LLVMContext &Ctx = Arg->getContext();
SmallVector<Metadata *, 2> ArgMDs{
MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)),
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 49830745ddd996..4ed448b4fe4e20 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -473,14 +473,19 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) {
// It's an internal service info from before IRTranslator passes.
MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
- for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
- if (MRI.getVRegDef(MI.getOperand(I).getReg()) != Def)
+ for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) {
+ MachineInstr *MaybeDef = MRI.getVRegDef(MI.getOperand(I).getReg());
+ if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
+ MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg());
+ if (MaybeDef != Def)
Def = nullptr;
+ }
if (Def) {
const MDNode *MD = MdMI->getOperand(1).getMetadata();
- Type *ValueTy = cast<ValueAsMetadata>(MD->getOperand(0))->getType();
StringRef ValueName =
cast<MDString>(MD->getOperand(1))->getString();
+ const MDNode *TypeMD = cast<MDNode>(MD->getOperand(0));
+ Type *ValueTy = getMDOperandAsType(TypeMD, 0);
GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName));
}
}
>From 1ff1d992aeb8ba613e1168b5fae51abb6d49f4b6 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 06:03:39 -0700
Subject: [PATCH 04/12] fixes
---
llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 10 +++-------
llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 7 +++++++
llvm/lib/Target/SPIRV/SPIRVUtils.h | 2 ++
3 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 4ed448b4fe4e20..74e309185e8067 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -472,14 +472,10 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
MachineInstr *MdMI = MI.getPrevNode();
if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) {
// It's an internal service info from before IRTranslator passes.
- MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
- for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) {
- MachineInstr *MaybeDef = MRI.getVRegDef(MI.getOperand(I).getReg());
- if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
- MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg());
- if (MaybeDef != Def)
+ MachineInstr *Def = getVRegDef(MRI, MI.getOperand(0).getReg());
+ for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
+ if (getVRegDef(MRI, MI.getOperand(I).getReg()) != Def)
Def = nullptr;
- }
if (Def) {
const MDNode *MD = MdMI->getOperand(1).getMetadata();
StringRef ValueName =
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 927683ad7e32be..9e715dac0677e0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -434,4 +434,11 @@ Type *parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx) {
return nullptr;
}
+MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
+ MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
+ if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
+ MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg());
+ return MaybeDef;
+}
+
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index c757af6b8aa72c..eadc4ce48930f8 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -238,5 +238,7 @@ inline const Type *unifyPtrType(const Type *Ty) {
return toTypedPointer(const_cast<Type *>(Ty));
}
+MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg);
+
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRVUTILS_H
>From bef9eda4b764ecd45e340cb30ec34031c9bbf6fe Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 08:37:34 -0700
Subject: [PATCH 05/12] implement 'Arithmetic with Overflow' intrinsics
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 17 +++++-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 60 -------------------
.../llvm-intrinsics/umul.with.overflow.ll | 4 ++
3 files changed, 20 insertions(+), 61 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 2e1dd1eeb6d729..070b08bc88caa3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1156,15 +1156,30 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
MIB.addUse(I.getOperand(i).getReg());
bool Status = MIB.constrainAllUses(TII, TRI, RBI);
// Build instructions to extract fields of the instruction's result.
+ // A new virtual register to store the higher part of the result struct.
+ Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+ MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass);
for (unsigned i = 0; i < I.getNumDefs(); ++i) {
auto MIB =
BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
- .addDef(I.getOperand(i).getReg())
+ .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg())
.addUse(GR.getSPIRVTypeID(ResType))
.addUse(StructVReg)
.addImm(i);
Status &= MIB.constrainAllUses(TII, TRI, RBI);
}
+ // Build boolean value from the higher part.
+ assert(I.getNumDefs() > 1 && "Not enought operands");
+ SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
+ unsigned N = GR.getScalarOrVectorComponentCount(ResType);
+ if (N > 1)
+ BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
+ Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
+ .addDef(I.getOperand(1).getReg())
+ .addUse(GR.getSPIRVTypeID(BoolType))
+ .addUse(HigherVReg)
+ .addUse(buildZerosVal(ResType, I))
+ .constrainAllUses(TII, TRI, RBI);
return Status;
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 78371485e0c669..25dd65710873b6 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -342,30 +342,6 @@ static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
FSHIntrinsic->setCalledFunction(FSHFunc);
}
-static void buildUMulWithOverflowFunc(Function *UMulFunc) {
- // The function body is already created.
- if (!UMulFunc->empty())
- return;
-
- BasicBlock *EntryBB = BasicBlock::Create(UMulFunc->getParent()->getContext(),
- "entry", UMulFunc);
- IRBuilder<> IRB(EntryBB);
- // Build the actual unsigned multiplication logic with the overflow
- // indication. Do unsigned multiplication Mul = A * B. Then check
- // if unsigned division Div = Mul / A is not equal to B. If so,
- // then overflow has happened.
- Value *Mul = IRB.CreateNUWMul(UMulFunc->getArg(0), UMulFunc->getArg(1));
- Value *Div = IRB.CreateUDiv(Mul, UMulFunc->getArg(0));
- Value *Overflow = IRB.CreateICmpNE(UMulFunc->getArg(0), Div);
-
- // umul.with.overflow intrinsic return a structure, where the first element
- // is the multiplication result, and the second is an overflow bit.
- Type *StructTy = UMulFunc->getReturnType();
- Value *Agg = IRB.CreateInsertValue(PoisonValue::get(StructTy), Mul, {0});
- Value *Res = IRB.CreateInsertValue(Agg, Overflow, {1});
- IRB.CreateRet(Res);
-}
-
static void lowerExpectAssume(IntrinsicInst *II) {
// If we cannot use the SPV_KHR_expect_assume extension, then we need to
// ignore the intrinsic and move on. It should be removed later on by LLVM.
@@ -407,20 +383,6 @@ static bool toSpvOverloadedIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID,
return true;
}
-static void lowerUMulWithOverflow(IntrinsicInst *UMulIntrinsic) {
- // Get a separate function - otherwise, we'd have to rework the CFG of the
- // current one. Then simply replace the intrinsic uses with a call to the new
- // function.
- Module *M = UMulIntrinsic->getModule();
- FunctionType *UMulFuncTy = UMulIntrinsic->getFunctionType();
- Type *FSHLRetTy = UMulFuncTy->getReturnType();
- const std::string FuncName = lowerLLVMIntrinsicName(UMulIntrinsic);
- Function *UMulFunc =
- getOrCreateFunction(M, FSHLRetTy, UMulFuncTy->params(), FuncName);
- buildUMulWithOverflowFunc(UMulFunc);
- UMulIntrinsic->setCalledFunction(UMulFunc);
-}
-
// Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
// or calls to proper generated functions. Returns True if F was modified.
bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
@@ -444,10 +406,6 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
lowerFunnelShifts(II);
Changed = true;
break;
-// case Intrinsic::umul_with_overflow:
-// lowerUMulWithOverflow(II);
-// Changed = true;
-// break;
case Intrinsic::assume:
case Intrinsic::expect: {
const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(*F);
@@ -482,24 +440,6 @@ SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
// Allow intrinsics with aggregate return type to reach GlobalISel
if (F->isIntrinsic() && IsRetAggr)
return F;
- /*
- // Allow intrinsics with aggregate return type to reach GlobalISel
- switch (F->getIntrinsicID()) {
- // Standard C/C++ Library Intrinsics
- case Intrinsic::frexp:
- // Vector Reduction Intrinsics
- case Intrinsic::vector_interleave2:
- case Intrinsic::vector_deinterleave2:
- // Arithmetic with Overflow Intrinsics
- case Intrinsic::smul_with_overflow:
- case Intrinsic::umul_with_overflow:
- case Intrinsic::sadd_with_overflow:
- case Intrinsic::uadd_with_overflow:
- case Intrinsic::ssub_with_overflow:
- case Intrinsic::usub_with_overflow:
- return F;
- }
-*/
IRBuilder<> B(F->getContext());
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
index c34771bf381ea9..047c8eff014205 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
@@ -1,5 +1,9 @@
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; This test fails due to change in code patterns that SPIR-V Backend emits for
+; "Arithmetic with Overflow" intrinsics.
+; XFAIL: *
+
; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_8:]] "spirv.llvm_umul_with_overflow_i8"
; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_32:]] "spirv.llvm_umul_with_overflow_i32"
; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_VEC_I64:]] "spirv.llvm_umul_with_overflow_v2i64"
>From 566cd75279afc63103cf10f037d19a02c21fe1f1 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 08:46:45 -0700
Subject: [PATCH 06/12] clang-format
---
llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 070b08bc88caa3..8f0a02af52405d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1175,11 +1175,11 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
if (N > 1)
BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
- .addDef(I.getOperand(1).getReg())
- .addUse(GR.getSPIRVTypeID(BoolType))
- .addUse(HigherVReg)
- .addUse(buildZerosVal(ResType, I))
- .constrainAllUses(TII, TRI, RBI);
+ .addDef(I.getOperand(1).getReg())
+ .addUse(GR.getSPIRVTypeID(BoolType))
+ .addUse(HigherVReg)
+ .addUse(buildZerosVal(ResType, I))
+ .constrainAllUses(TII, TRI, RBI);
return Status;
}
>From 35bde11a463a36829875f60058bc429537f6bd17 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 13:42:38 -0700
Subject: [PATCH 07/12] add a test case
---
.../llvm-intrinsics/arith.with.overflow.ll | 89 +++++++++++++++++++
1 file changed, 89 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll
new file mode 100644
index 00000000000000..7113dd692f6ac4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll
@@ -0,0 +1,89 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+ %umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b)
+ %cmp = extractvalue { i8, i1 } %umul, 1
+ %umul.value = extractvalue { i8, i1 } %umul, 0
+ %storemerge = select i1 %cmp, i8 0, i8 %umul.value
+ store i8 %storemerge, ptr %c, align 1
+ ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+ %umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a)
+ %umul.val = extractvalue { i32, i1 } %umul, 0
+ %umul.ov = extractvalue { i32, i1 } %umul, 1
+ %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
+ store i32 %spec.select, ptr %c, align 4
+ ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
+ %umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+ %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
+ %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
+ %zero = alloca <2 x i64>, align 16
+ %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
+ store <2 x i64> %spec.select, ptr %p
+ ret void
+}
+
+declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
>From 5fb69011d896ce8a1a15618c6c454c865e831ae3 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 17 Sep 2024 14:04:16 -0700
Subject: [PATCH 08/12] add scalar/vector versions of OpIAddCarry/OpISubBorrow;
add test cases
---
llvm/lib/Target/SPIRV/SPIRVInstrInfo.td | 4 +-
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 10 +-
...with.overflow.ll => smul.with.overflow.ll} | 18 ++--
.../llvm-intrinsics/uadd.with.overflow.ll | 89 ++++++++++++++++++
.../llvm-intrinsics/umul.with.overflow.ll | 91 +++++++++++++------
.../llvm-intrinsics/usub.with.overflow.ll | 89 ++++++++++++++++++
6 files changed, 258 insertions(+), 43 deletions(-)
rename llvm/test/CodeGen/SPIRV/llvm-intrinsics/{arith.with.overflow.ll => smul.with.overflow.ll} (87%)
create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
create mode 100644 llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 0cbd2f4536075d..9dbc50247ccf3e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -519,8 +519,8 @@ def OpMatrixTimesMatrix: BinOp<"OpMatrixTimesMatrix", 146>;
def OpOuterProduct: BinOp<"OpOuterProduct", 147>;
def OpDot: BinOp<"OpDot", 148>;
-def OpIAddCarry: BinOpTyped<"OpIAddCarry", 149, iID, addc>;
-def OpISubBorrow: BinOpTyped<"OpISubBorrow", 150, iID, subc>;
+defm OpIAddCarry: BinOpTypedGen<"OpIAddCarry", 149, addc, 0, 1>;
+defm OpISubBorrow: BinOpTypedGen<"OpISubBorrow", 150, subc, 0, 1>;
def OpUMulExtended: BinOp<"OpUMulExtended", 151>;
def OpSMulExtended: BinOp<"OpSMulExtended", 152>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 8f0a02af52405d..a8e3ae21e874ad 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -598,13 +598,19 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
"Selection of a signed addition with overflow is not implemented",
false);
case TargetOpcode::G_UADDO:
- return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpIAddCarry);
+ return selectOverflowArith(ResVReg, ResType, I,
+ ResType->getOpcode() == SPIRV::OpTypeVector
+ ? SPIRV::OpIAddCarryV
+ : SPIRV::OpIAddCarryS);
case TargetOpcode::G_SSUBO:
report_fatal_error(
"Selection of a signed subtraction with overflow is not implemented",
false);
case TargetOpcode::G_USUBO:
- return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpISubBorrow);
+ return selectOverflowArith(ResVReg, ResType, I,
+ ResType->getOpcode() == SPIRV::OpTypeVector
+ ? SPIRV::OpISubBorrowV
+ : SPIRV::OpISubBorrowS);
case TargetOpcode::G_UMULO:
return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
case TargetOpcode::G_SMULO:
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll
similarity index 87%
rename from llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll
rename to llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll
index 7113dd692f6ac4..2281ccf52bbb4e 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/arith.with.overflow.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll
@@ -25,7 +25,7 @@
; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
-; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Struct:.*]] = OpSMulExtended %[[StructChar]] %[[A]] %[[B]]
; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
@@ -34,7 +34,7 @@
; CHECK: OpReturn
define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
entry:
- %umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b)
+ %umul = tail call { i8, i1 } @llvm.smul.with.overflow.i8(i8 %a, i8 %b)
%cmp = extractvalue { i8, i1 } %umul, 1
%umul.value = extractvalue { i8, i1 } %umul, 0
%storemerge = select i1 %cmp, i8 0, i8 %umul.value
@@ -46,7 +46,7 @@ entry:
; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
-; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Struct2:.*]] = OpSMulExtended %[[StructInt]] %[[B2]] %[[A2]]
; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
@@ -55,7 +55,7 @@ entry:
; CHECK: OpReturn
define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
entry:
- %umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a)
+ %umul = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %b, i32 %a)
%umul.val = extractvalue { i32, i1 } %umul, 0
%umul.ov = extractvalue { i32, i1 } %umul, 1
%spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
@@ -67,7 +67,7 @@ entry:
; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
-; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Struct3:.*]] = OpSMulExtended %[[StructV2Long]] %[[A3]] %[[B3]]
; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
@@ -75,7 +75,7 @@ entry:
; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
; CHECK: OpReturn
define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
- %umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+ %umul = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
%umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
%umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
%zero = alloca <2 x i64>, align 16
@@ -84,6 +84,6 @@ define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p)
ret void
}
-declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8)
-declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32)
-declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {i8, i1} @llvm.smul.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.smul.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
new file mode 100644
index 00000000000000..cecd6f60655dcf
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
@@ -0,0 +1,89 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpIAddCarry %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+ %umul = tail call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
+ %cmp = extractvalue { i8, i1 } %umul, 1
+ %umul.value = extractvalue { i8, i1 } %umul, 0
+ %storemerge = select i1 %cmp, i8 0, i8 %umul.value
+ store i8 %storemerge, ptr %c, align 1
+ ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpIAddCarry %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+ %umul = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %b, i32 %a)
+ %umul.val = extractvalue { i32, i1 } %umul, 0
+ %umul.ov = extractvalue { i32, i1 } %umul, 1
+ %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
+ store i32 %spec.select, ptr %c, align 4
+ ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpIAddCarry %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
+ %umul = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+ %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
+ %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
+ %zero = alloca <2 x i64>, align 16
+ %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
+ store <2 x i64> %spec.select, ptr %p
+ ret void
+}
+
+declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
index 047c8eff014205..7113dd692f6ac4 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
@@ -1,58 +1,89 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; This test fails due to change in code patterns that SPIR-V Backend emits for
-; "Arithmetic with Overflow" intrinsics.
-; XFAIL: *
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_8:]] "spirv.llvm_umul_with_overflow_i8"
-; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_32:]] "spirv.llvm_umul_with_overflow_i32"
-; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_VEC_I64:]] "spirv.llvm_umul_with_overflow_v2i64"
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
-define dso_local spir_func void @_Z4foo8hhPh(i8 zeroext %a, i8 zeroext %b, i8* nocapture %c) local_unnamed_addr {
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
entry:
- ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_8]]
%umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b)
%cmp = extractvalue { i8, i1 } %umul, 1
%umul.value = extractvalue { i8, i1 } %umul, 0
%storemerge = select i1 %cmp, i8 0, i8 %umul.value
- store i8 %storemerge, i8* %c, align 1
+ store i8 %storemerge, ptr %c, align 1
ret void
}
-define dso_local spir_func void @_Z5foo32jjPj(i32 %a, i32 %b, i32* nocapture %c) local_unnamed_addr {
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
entry:
- ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_32]]
%umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a)
%umul.val = extractvalue { i32, i1 } %umul, 0
%umul.ov = extractvalue { i32, i1 } %umul, 1
%spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
- store i32 %spec.select, i32* %c, align 4
+ store i32 %spec.select, ptr %c, align 4
ret void
}
-define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64>* %p) nounwind {
- ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_VEC_I64]]
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
%umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
%umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
%umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
%zero = alloca <2 x i64>, align 16
%spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
- store <2 x i64> %spec.select, <2 x i64>* %p
+ store <2 x i64> %spec.select, ptr %p
ret void
}
-; CHECK-SPIRV: %[[#NAME_UMUL_FUNC_8]] = OpFunction %[[#]]
-; CHECK-SPIRV: %[[#VAR_A:]] = OpFunctionParameter %[[#]]
-; CHECK-SPIRV: %[[#VAR_B:]] = OpFunctionParameter %[[#]]
-; CHECK-SPIRV: %[[#MUL_RES:]] = OpIMul %[[#]] %[[#VAR_A]] %[[#VAR_B]]
-; CHECK-SPIRV: %[[#DIV_RES:]] = OpUDiv %[[#]] %[[#MUL_RES]] %[[#VAR_A]]
-; CHECK-SPIRV: %[[#CMP_RES:]] = OpINotEqual %[[#]] %[[#VAR_A]] %[[#DIV_RES]]
-; CHECK-SPIRV: %[[#INSERT_RES:]] = OpCompositeInsert %[[#]] %[[#MUL_RES]]
-; CHECK-SPIRV: %[[#INSERT_RES_1:]] = OpCompositeInsert %[[#]] %[[#CMP_RES]] %[[#INSERT_RES]]
-; CHECK-SPIRV: OpReturnValue %[[#INSERT_RES_1]]
-
-declare { i8, i1 } @llvm.umul.with.overflow.i8(i8, i8)
-
-declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32)
-
+declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32)
declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll
new file mode 100644
index 00000000000000..963dd70f606b6e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll
@@ -0,0 +1,89 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpISubBorrow %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+ %umul = tail call { i8, i1 } @llvm.usub.with.overflow.i8(i8 %a, i8 %b)
+ %cmp = extractvalue { i8, i1 } %umul, 1
+ %umul.value = extractvalue { i8, i1 } %umul, 0
+ %storemerge = select i1 %cmp, i8 0, i8 %umul.value
+ store i8 %storemerge, ptr %c, align 1
+ ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpISubBorrow %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+ %umul = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %b, i32 %a)
+ %umul.val = extractvalue { i32, i1 } %umul, 0
+ %umul.ov = extractvalue { i32, i1 } %umul, 1
+ %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
+ store i32 %spec.select, ptr %c, align 4
+ ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpISubBorrow %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
+ %umul = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+ %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
+ %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
+ %zero = alloca <2 x i64>, align 16
+ %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
+ store <2 x i64> %spec.select, ptr %p
+ ret void
+}
+
+declare {i8, i1} @llvm.usub.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.usub.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64>, <2 x i64>)
>From bde81a3f3654e5039e8434feb6e6dece573522f6 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 18 Sep 2024 01:31:00 -0700
Subject: [PATCH 09/12] update docs
---
llvm/docs/SPIRVUsage.rst | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 0f0b21fb237703..9e53bf66d0e627 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -275,6 +275,10 @@ SPIR-V backend, along with their descriptions and argument details.
- None
- `[Type, Vararg]`
- Assigns names to types or values, enhancing readability and debuggability of SPIR-V code. Not emitted directly but used for metadata enrichment.
+ * - `int_spv_value_md`
+ - None
+ - `[Metadata]`
+ - Assigns a set of attributes (such as name and data type) to a value that is an argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value.
* - `int_spv_assign_decoration`
- None
- `[Type, Metadata]`
>From ae5fe0e32b9940e3946f6df88cd2ab4aaaad8497 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 18 Sep 2024 02:24:32 -0700
Subject: [PATCH 10/12] add specific test cases: (1) example of llvm intrinsics
originated from an optimization pass; (2) key translation steps with checks
of LLVM IR / gMIR / SPIRV code patterns
---
.../SPIRV/optimizations/add-check-overflow.ll | 56 ++++++++++++++++
.../SPIRV/passes/translate-aggregate-uaddo.ll | 64 +++++++++++++++++++
2 files changed, 120 insertions(+)
create mode 100644 llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll
create mode 100644 llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll
diff --git a/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll
new file mode 100644
index 00000000000000..22c51a56057966
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll
@@ -0,0 +1,56 @@
+; This test aims to check ability to support "Arithmetic with Overflow" intrinsics
+; in the special case when those intrinsics are being generated by the CodeGenPrepare;
+; pass during translations with optimization (note -O3 in llc arguments).
+
+; RUN: llc -O3 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O3 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[Val:.*]] "math"
+; CHECK-DAG: OpName %[[IsOver:.*]] "ov"
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Generic %[[Char]]
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[Struct:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[Const1:.*]] = OpConstant %[[Int]] 1
+; CHECK-DAG: %[[Const42:.*]] = OpConstant %[[Char]] 42
+; CHECK-DAG: %[[Zero:.*]] = OpConstantNull %[[Int]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[#]] = OpLabel
+; CHECK: OpBranch %[[#]]
+; CHECK: %[[#]] = OpLabel
+; CHECK: %[[PhiRes:.*]] = OpPhi %[[Int]] %[[A]] %[[#]] %[[Val]] %[[#]]
+; CHECK: %[[AggRes:.*]] = OpIAddCarry %[[Struct]] %[[PhiRes]] %[[Const1]]
+; CHECK: %[[Val]] = OpCompositeExtract %[[Int]] %[[AggRes]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Int]] %[[AggRes]] 1
+; CHECK: %[[IsOver]] = OpINotEqual %[[Bool:.*]] %[[Over]] %[[Zero]]
+; CHECK: OpBranchConditional %[[IsOver]] %[[#]] %[[#]]
+; CHECK: OpStore %[[Ptr]] %[[Const42]] Aligned 1
+; CHECK: OpBranch %[[#]]
+; CHECK: %[[#]] = OpLabel
+; CHECK: OpReturnValue %[[Val]]
+; CHECK: OpFunctionEnd
+
+define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) {
+entry:
+ br label %l1
+
+l1:
+ %e = phi i32 [ %a, %entry ], [ %i, %body ]
+ %i = add nsw i32 %e, 1
+ %fl = icmp eq i32 %i, 0
+ br i1 %fl, label %exit, label %body
+
+body:
+ store i8 42, ptr addrspace(4) %p
+ br label %l1
+
+exit:
+ ret i32 %i
+}
diff --git a/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll b/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll
new file mode 100644
index 00000000000000..cd4d9325c76599
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll
@@ -0,0 +1,64 @@
+; This test shows how value attributes are being passed during different translation steps.
+; See also test/CodeGen/SPIRV/optimizations/add-check-overflow.ll.
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=prepare-functions 2>&1 | FileCheck %s --check-prefix=CHECK-PREPARE
+; Intrinsics with aggregate return type are not substituted/removed.
+; CHECK-PREPARE: @llvm.uadd.with.overflow.i32
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=emit-intrinsics 2>&1 | FileCheck %s --check-prefix=CHECK-IR
+; Aggregate data are wrapped into @llvm.fake.use(),
+; and their attributes are packed into a metadata for @llvm.spv.value.md().
+; CHECK-IR: %[[R1:.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32
+; CHECK-IR: call void @llvm.spv.value.md(metadata !0)
+; CHECK-IR: call void (...) @llvm.fake.use({ i32, i1 } %[[R1]])
+; CHECK-IR: %math = extractvalue { i32, i1 } %[[R1]], 0
+; CHECK-IR: %ov = extractvalue { i32, i1 } %[[R1]], 1
+; Type/Name attributes of the value.
+; CHECK-IR: !0 = !{{[{]}}!1, !""{{[}]}}
+; Origin data type of the value.
+; CHECK-IR: !1 = !{{[{]}}{{[{]}} i32, i1 {{[}]}} poison{{[}]}}
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=irtranslator 2>&1 | FileCheck %s --check-prefix=CHECK-GMIR
+; Required info succeeded to get through IRTranslator.
+; CHECK-GMIR: %[[phires:.*]]:_(s32) = G_PHI
+; CHECK-GMIR: %[[math:.*]]:id(s32), %[[ov:.*]]:_(s1) = G_UADDO %[[phires]]:_, %[[#]]:_
+; CHECK-GMIR: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.spv.value.md), !0
+; CHECK-GMIR: FAKE_USE %[[math]]:id(s32), %[[ov]]:_(s1)
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=spirv-prelegalizer 2>&1 | FileCheck %s --check-prefix=CHECK-PRE
+; Internal service instructions are consumed.
+; CHECK-PRE: G_UADDO
+; CHECK-PRE-NO: llvm.spv.value.md
+; CHECK-PRE-NO: FAKE_USE
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=instruction-select 2>&1 | FileCheck %s --check-prefix=CHECK-ISEL
+; Names and types are restored and correctly encoded. Correct instruction selection is completed.
+; CHECK-ISEL-DAG: %[[int32:.*]]:type = OpTypeInt 32, 0
+; CHECK-ISEL-DAG: %[[struct:.*]]:type = OpTypeStruct %[[int32]]:type, %[[int32]]:type
+; CHECK-ISEL-DAG: %[[bool:.*]]:type = OpTypeBool
+; CHECK-ISEL-DAG: %[[zero32:.*]]:iid = OpConstantNull %[[int32]]:type
+; CHECK-ISEL-DAG: %[[res:.*]]:iid = OpIAddCarryS %[[struct]]:type
+; CHECK-ISEL-DAG: %[[math:.*]]:id = OpCompositeExtract %[[int32]]:type, %[[res]]:iid, 0
+; CHECK-ISEL-DAG: %[[ov32:.*]]:iid = OpCompositeExtract %[[int32]]:type, %[[res]]:iid, 1
+; CHECK-ISEL-DAG: %[[ov:.*]]:iid = OpINotEqual %[[bool]]:type, %[[ov32]]:iid, %[[zero32:.*]]:iid
+; CHECK-ISEL-DAG: OpName %[[math]]:id, 1752457581, 0
+; CHECK-ISEL-DAG: OpName %[[ov]]:iid, 30319
+
+define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) {
+entry:
+ br label %l1
+
+l1: ; preds = %body, %entry
+ %e = phi i32 [ %a, %entry ], [ %math, %body ]
+ %0 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %e, i32 1)
+ %math = extractvalue { i32, i1 } %0, 0
+ %ov = extractvalue { i32, i1 } %0, 1
+ br i1 %ov, label %exit, label %body
+
+body: ; preds = %l1
+ store i8 42, ptr addrspace(4) %p, align 1
+ br label %l1
+
+exit: ; preds = %l1
+ ret i32 %math
+}
>From 27f9f5c3de06e75606423847cda1f4219daa29af Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 18 Sep 2024 03:12:09 -0700
Subject: [PATCH 11/12] correct insertion points for new instructions
---
llvm/docs/SPIRVUsage.rst | 2 +-
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 25 +++++++++++--------
2 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 9e53bf66d0e627..bb12b05246afba 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -278,7 +278,7 @@ SPIR-V backend, along with their descriptions and argument details.
* - `int_spv_value_md`
- None
- `[Metadata]`
- - Assigns a set of attributes (such as name and data type) to a value that is an argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value.
+ - Assigns a set of attributes (such as name and data type) to a value that is the argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value.
* - `int_spv_assign_decoration`
- None
- `[Type, Metadata]`
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index a8e3ae21e874ad..fb84edccf87fff 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -1143,10 +1143,17 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
// and the two members must be the same type."
Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy});
- // Build SPIR-V type if needed.
+ // Build SPIR-V types and constant(s) if needed.
MachineIRBuilder MIRBuilder(I);
SPIRVType *StructType = GR.getOrCreateSPIRVType(
ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
+ assert(I.getNumDefs() > 1 && "Not enought operands");
+ SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
+ unsigned N = GR.getScalarOrVectorComponentCount(ResType);
+ if (N > 1)
+ BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
+ Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
+ Register ZeroReg = buildZerosVal(ResType, I);
// A new virtual register to store the result struct.
Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
@@ -1155,9 +1162,10 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
buildOpName(StructVReg, ResName, MIRBuilder);
// Build the arithmetic with overflow instruction.
MachineBasicBlock &BB = *I.getParent();
- auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
- .addDef(StructVReg)
- .addUse(GR.getSPIRVTypeID(StructType));
+ auto MIB =
+ BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode))
+ .addDef(StructVReg)
+ .addUse(GR.getSPIRVTypeID(StructType));
for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
MIB.addUse(I.getOperand(i).getReg());
bool Status = MIB.constrainAllUses(TII, TRI, RBI);
@@ -1175,16 +1183,11 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
Status &= MIB.constrainAllUses(TII, TRI, RBI);
}
// Build boolean value from the higher part.
- assert(I.getNumDefs() > 1 && "Not enought operands");
- SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
- unsigned N = GR.getScalarOrVectorComponentCount(ResType);
- if (N > 1)
- BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
.addDef(I.getOperand(1).getReg())
- .addUse(GR.getSPIRVTypeID(BoolType))
+ .addUse(BoolTypeReg)
.addUse(HigherVReg)
- .addUse(buildZerosVal(ResType, I))
+ .addUse(ZeroReg)
.constrainAllUses(TII, TRI, RBI);
return Status;
}
>From 788757a4bf1be68da016cbc90254355325635146 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Thu, 19 Sep 2024 02:56:04 -0700
Subject: [PATCH 12/12] apply code review suggestions
---
.../lib/Target/SPIRV/SPIRVInstructionSelector.cpp | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index fb84edccf87fff..1a6ad4d2635abc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -412,11 +412,22 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) {
return false;
}
+static bool mayApplyGenericSelection(unsigned Opcode) {
+ switch (Opcode) {
+ case TargetOpcode::G_CONSTANT:
+ return false;
+ case TargetOpcode::G_SADDO:
+ case TargetOpcode::G_SSUBO:
+ return true;
+ }
+ return isTypeFoldingSupported(Opcode);
+}
+
bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
const SPIRVType *ResType,
MachineInstr &I) const {
const unsigned Opcode = I.getOpcode();
- if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT)
+ if (mayApplyGenericSelection(Opcode))
return selectImpl(I, *CoverageInfo);
switch (Opcode) {
case TargetOpcode::G_CONSTANT:
@@ -1157,7 +1168,7 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
// A new virtual register to store the result struct.
Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
- // Build the result name of needed.
+ // Build the result name if needed.
if (ResName.size() > 0)
buildOpName(StructVReg, ResName, MIRBuilder);
// Build the arithmetic with overflow instruction.
More information about the llvm-commits
mailing list