[llvm] 1aa8a6f - [VPlan] Compute cost for most opcodes in VPWidenRecipe (NFCI). (#98764)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 16 12:20:27 PDT 2024
Author: Florian Hahn
Date: 2024-08-16T21:20:23+02:00
New Revision: 1aa8a6f6917edbdcad53afad3b00c7e86aa57ffe
URL: https://github.com/llvm/llvm-project/commit/1aa8a6f6917edbdcad53afad3b00c7e86aa57ffe
DIFF: https://github.com/llvm/llvm-project/commit/1aa8a6f6917edbdcad53afad3b00c7e86aa57ffe.diff
LOG: [VPlan] Compute cost for most opcodes in VPWidenRecipe (NFCI). (#98764)
Implement VPWidenRecipe::computeCost for most cases (except
UDiv,SDiv,URem,SRem which require additional logic).
Note that this specializes `::computeCost` instead of `::cost`, as
`VPRecipeBase::cost` is responsible for skipping cost-computations
for pre-computed recipes for now.
The most recent version of the VPlan-based cost model introduction
has been committed on Jul 10 (b841e2eca3b5c8b) and we should
probably give it at least a week in case additional mismatches surface.
PR: https://github.com/llvm/llvm-project/pull/98764
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 58ef665e86d65e..27e99694aafd30 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4354,8 +4354,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
SmallVector<RecipeVFPair> InvalidCosts;
for (const auto &Plan : VPlans) {
for (ElementCount VF : Plan->vectorFactors()) {
- VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx,
- CM);
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+ LLVMCtx, CM);
auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
for (auto &R : *VPBB) {
@@ -7062,7 +7062,8 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
ElementCount VF) const {
InstructionCost Cost = 0;
LLVMContext &LLVMCtx = OrigLoop->getHeader()->getContext();
- VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx, CM);
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+ LLVMCtx, CM);
// Cost modeling for inductions is inaccurate in the legacy cost model
// compared to the recipes that are generated. To match here initially during
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 016ad75c21ceb0..a99f3882092c2c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -736,14 +736,16 @@ class VPLiveOut : public VPUser {
/// Struct to hold various analysis needed for cost computations.
struct VPCostContext {
const TargetTransformInfo &TTI;
+ const TargetLibraryInfo &TLI;
VPTypeAnalysis Types;
LLVMContext &LLVMCtx;
LoopVectorizationCostModel &CM;
SmallPtrSet<Instruction *, 8> SkipCostComputation;
- VPCostContext(const TargetTransformInfo &TTI, Type *CanIVTy,
- LLVMContext &LLVMCtx, LoopVectorizationCostModel &CM)
- : TTI(TTI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
+ VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+ Type *CanIVTy, LLVMContext &LLVMCtx,
+ LoopVectorizationCostModel &CM)
+ : TTI(TTI), TLI(TLI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
/// Return the cost for \p UI with \p VF using the legacy cost model as
/// fallback until computing the cost of all recipes migrates to VPlan.
@@ -796,7 +798,7 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
/// Return the cost of this recipe, taking into account if the cost
/// computation should be skipped and the ForceTargetInstructionCost flag.
/// Also takes care of printing the cost for debugging.
- virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
+ InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
/// Insert an unlinked recipe into a basic block immediately before
/// the specified recipe.
@@ -860,9 +862,11 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
DebugLoc getDebugLoc() const { return DL; }
protected:
- /// Compute the cost of this recipe using the legacy cost model and the
- /// underlying instructions.
- InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
+ /// Compute the cost of this recipe either using a recipe's specialized
+ /// implementation or using the legacy cost model and the underlying
+ /// instructions.
+ virtual InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const;
};
// Helper macro to define common classof implementations for recipes.
@@ -1426,6 +1430,10 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
/// processing State.VF elements.
void execute(VPTransformState &State) override;
+ /// Return the cost of this VPWidenRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
unsigned getOpcode() const { return Opcode; }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bc57ea4d52471e..c9d603612aecea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1140,6 +1140,80 @@ void VPWidenRecipe::execute(VPTransformState &State) {
#endif
}
+InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ switch (Opcode) {
+ case Instruction::FNeg: {
+ Type *VectorTy =
+ ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+ return Ctx.TTI.getArithmeticInstrCost(
+ Opcode, VectorTy, CostKind,
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});
+ }
+
+ case Instruction::UDiv:
+ case Instruction::SDiv:
+ case Instruction::SRem:
+ case Instruction::URem:
+ // More complex computation, let the legacy cost-model handle this for now.
+ return Ctx.getLegacyCost(cast<Instruction>(getUnderlyingValue()), VF);
+ case Instruction::Add:
+ case Instruction::FAdd:
+ case Instruction::Sub:
+ case Instruction::FSub:
+ case Instruction::Mul:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ case Instruction::Shl:
+ case Instruction::LShr:
+ case Instruction::AShr:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor: {
+ VPValue *RHS = getOperand(1);
+ // Certain instructions can be cheaper to vectorize if they have a constant
+ // second vector operand. One example of this are shifts on x86.
+ TargetTransformInfo::OperandValueInfo RHSInfo = {
+ TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None};
+ if (RHS->isLiveIn())
+ RHSInfo = Ctx.TTI.getOperandInfo(RHS->getLiveInIRValue());
+
+ if (RHSInfo.Kind == TargetTransformInfo::OK_AnyValue &&
+ getOperand(1)->isDefinedOutsideVectorRegions())
+ RHSInfo.Kind = TargetTransformInfo::OK_UniformValue;
+ Type *VectorTy =
+ ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+ Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+
+ SmallVector<const Value *, 4> Operands;
+ if (CtxI)
+ Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
+ return Ctx.TTI.getArithmeticInstrCost(
+ Opcode, VectorTy, CostKind,
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+ RHSInfo, Operands, CtxI, &Ctx.TLI);
+ }
+ case Instruction::Freeze: {
+ // This opcode is unknown. Assume that it is the same as 'mul'.
+ Type *VectorTy =
+ ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+ return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
+ }
+ case Instruction::ICmp:
+ case Instruction::FCmp: {
+ Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+ Type *VectorTy = ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+ return Ctx.TTI.getCmpSelInstrCost(Opcode, VectorTy, nullptr, getPredicate(),
+ CostKind, CtxI);
+ }
+ default:
+ llvm_unreachable("Unsupported opcode for instruction");
+ }
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
More information about the llvm-commits
mailing list