[llvm] b841e2e - Recommit "[VPlan] First step towards VPlan cost modeling. (#92555)"
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 10 06:22:46 PDT 2024
Author: Florian Hahn
Date: 2024-07-10T14:22:21+01:00
New Revision: b841e2eca3b5c8b408214a46593f6a025e0fe48b
URL: https://github.com/llvm/llvm-project/commit/b841e2eca3b5c8b408214a46593f6a025e0fe48b
DIFF: https://github.com/llvm/llvm-project/commit/b841e2eca3b5c8b408214a46593f6a025e0fe48b.diff
LOG: Recommit "[VPlan] First step towards VPlan cost modeling. (#92555)"
This reverts commit 6f538f6a2d3224efda985e9eb09012fa4275ea92.
A number of crashes have been fixed by separate fixes, including
ttps://github.com/llvm/llvm-project/pull/96622. This version of the
PR also pre-computes the costs for branches (except the latch) instead
of computing their costs as part of costing of replicate regions, as
there may not be a direct correspondence between original branches and
number of replicate regions.
Original message:
This adds a new interface to compute the cost of recipes, VPBasicBlocks,
VPRegionBlocks and VPlan, initially falling back to the legacy cost model
for all recipes. Follow-up patches will gradually migrate recipes to
compute their own costs step-by-step.
It also adds getBestPlan function to LVP which computes the cost of all
VPlans and picks the most profitable one together with the most
profitable VF.
The VPlan selected by the VPlan cost model is executed and there is an
assert to catch cases where the VPlan cost model and the legacy cost
model disagree. Even though I checked a number of different build
configurations on AArch64 and X86, there may be some differences
that have been missed.
Additional discussions and context can be found in @arcbbb's
https://github.com/llvm/llvm-project/pull/67647 and
https://github.com/llvm/llvm-project/pull/67934 which is an earlier
version of the current PR.
PR: https://github.com/llvm/llvm-project/pull/92555
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanValue.h
llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/x86-loopvectorize-costmodel.ll.expected
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 44db0ea33dafb..c63cf0c37f2f9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -334,6 +334,16 @@ class LoopVectorizationPlanner {
/// A builder used to construct the current plan.
VPBuilder Builder;
+ /// Computes the cost of \p Plan for vectorization factor \p VF.
+ ///
+ /// The current implementation requires access to the
+ /// LoopVectorizationLegality to handle inductions and reductions, which is
+ /// why it is kept separate from the VPlan-only cost infrastructure.
+ ///
+ /// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
+ /// been retired.
+ InstructionCost cost(VPlan &Plan, ElementCount VF) const;
+
public:
LoopVectorizationPlanner(
Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
@@ -355,6 +365,9 @@ class LoopVectorizationPlanner {
/// Return the best VPlan for \p VF.
VPlan &getBestPlanFor(ElementCount VF) const;
+ /// Return the most profitable plan and fix its VF to the most profitable one.
+ VPlan &getBestPlan() const;
+
/// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan
/// according to the best selected \p VF and \p UF.
///
@@ -434,6 +447,9 @@ class LoopVectorizationPlanner {
/// \return The most profitable vectorization factor for the available VPlans
/// and the cost of that VF.
+ /// This is now only used to verify the decisions by the new VPlan-based
+ /// cost-model and will be retired once the VPlan-based cost-model is
+ /// stabilized.
VectorizationFactor selectVectorizationFactor();
/// Returns true if the per-lane cost of VectorizationFactor A is lower than
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9749f3db76489..7b2fab1f8a741 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -290,7 +290,7 @@ static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
cl::desc("A flag that overrides the target's max interleave factor for "
"vectorized loops."));
-static cl::opt<unsigned> ForceTargetInstructionCost(
+cl::opt<unsigned> ForceTargetInstructionCost(
"force-target-instruction-cost", cl::init(0), cl::Hidden,
cl::desc("A flag that overrides the target's expected cost for "
"an instruction to a single constant value. Mostly "
@@ -412,14 +412,6 @@ static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
}
-/// A helper function that returns the reciprocal of the block probability of
-/// predicated blocks. If we return X, we are assuming the predicated block
-/// will execute once for every X iterations of the loop header.
-///
-/// TODO: We should use actual block probability here, if available. Currently,
-/// we always assume predicated blocks have a 50% chance of executing.
-static unsigned getReciprocalPredBlockProb() { return 2; }
-
/// Returns "best known" trip count for the specified loop \p L as defined by
/// the following procedure:
/// 1) Returns exact trip count if it is known.
@@ -1608,6 +1600,16 @@ class LoopVectorizationCostModel {
/// \p VF is the vectorization factor chosen for the original loop.
bool isEpilogueVectorizationProfitable(const ElementCount VF) const;
+ /// Returns the execution time cost of an instruction for a given vector
+ /// width. Vector width of one means scalar.
+ InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
+
+ /// Return the cost of instructions in an inloop reduction pattern, if I is
+ /// part of that pattern.
+ std::optional<InstructionCost>
+ getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy,
+ TTI::TargetCostKind CostKind) const;
+
private:
unsigned NumPredStores = 0;
@@ -1633,16 +1635,6 @@ class LoopVectorizationCostModel {
/// of elements.
ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
- /// Returns the execution time cost of an instruction for a given vector
- /// width. Vector width of one means scalar.
- InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
-
- /// Return the cost of instructions in an inloop reduction pattern, if I is
- /// part of that pattern.
- std::optional<InstructionCost>
- getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy,
- TTI::TargetCostKind CostKind) const;
-
/// Calculate vectorization cost of memory instruction \p I.
InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
@@ -7288,7 +7280,10 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
[](std::unique_ptr<VPlan> &P) { return P->hasScalarVFOnly(); }))
return VectorizationFactor::Disabled();
- // Select the optimal vectorization factor.
+ // Select the optimal vectorization factor according to the legacy cost-model.
+ // This is now only used to verify the decisions by the new VPlan-based
+ // cost-model and will be retired once the VPlan-based cost-model is
+ // stabilized.
VectorizationFactor VF = selectVectorizationFactor();
assert((VF.Width.isScalar() || VF.ScalarCost > 0) && "when vectorizing, the scalar cost must be non-zero.");
if (!hasPlanWithVF(VF.Width)) {
@@ -7299,6 +7294,211 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
return VF;
}
+InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
+ ElementCount VF) const {
+ return CM.getInstructionCost(UI, VF);
+}
+
+bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
+ return CM.ValuesToIgnore.contains(UI) ||
+ (IsVector && CM.VecValuesToIgnore.contains(UI)) ||
+ SkipCostComputation.contains(UI);
+}
+
+InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
+ ElementCount VF) const {
+ InstructionCost Cost = 0;
+ LLVMContext &LLVMCtx = OrigLoop->getHeader()->getContext();
+ VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx, CM);
+
+ // Cost modeling for inductions is inaccurate in the legacy cost model
+ // compared to the recipes that are generated. To match here initially during
+ // VPlan cost model bring up directly use the induction costs from the legacy
+ // cost model. Note that we do this as pre-processing; the VPlan may not have
+ // any recipes associated with the original induction increment instruction
+ // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
+ // the cost of induction phis and increments (both that are represented by
+ // recipes and those that are not), to avoid distinguishing between them here,
+ // and skip all recipes that represent induction phis and increments (the
+ // former case) later on, if they exist, to avoid counting them twice.
+ // Similarly we pre-compute the cost of any optimized truncates.
+ // TODO: Switch to more accurate costing based on VPlan.
+ for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
+ Instruction *IVInc = cast<Instruction>(
+ IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
+ SmallVector<Instruction *> IVInsts = {IV, IVInc};
+ for (User *U : IV->users()) {
+ auto *CI = cast<Instruction>(U);
+ if (!CostCtx.CM.isOptimizableIVTruncate(CI, VF))
+ continue;
+ IVInsts.push_back(CI);
+ }
+ for (Instruction *IVInst : IVInsts) {
+ if (!CostCtx.SkipCostComputation.insert(IVInst).second)
+ continue;
+ InstructionCost InductionCost = CostCtx.getLegacyCost(IVInst, VF);
+ LLVM_DEBUG({
+ dbgs() << "Cost of " << InductionCost << " for VF " << VF
+ << ": induction instruction " << *IVInst << "\n";
+ });
+ Cost += InductionCost;
+ }
+ }
+
+ /// Compute the cost of all exiting conditions of the loop using the legacy
+ /// cost model. This is to match the legacy behavior, which adds the cost of
+ /// all exit conditions. Note that this over-estimates the cost, as there will
+ /// be a single condition to control the vector loop.
+ SmallVector<BasicBlock *> Exiting;
+ CM.TheLoop->getExitingBlocks(Exiting);
+ SetVector<Instruction *> ExitInstrs;
+ // Collect all exit conditions.
+ for (BasicBlock *EB : Exiting) {
+ auto *Term = dyn_cast<BranchInst>(EB->getTerminator());
+ if (!Term)
+ continue;
+ if (auto *CondI = dyn_cast<Instruction>(Term->getOperand(0))) {
+ ExitInstrs.insert(CondI);
+ }
+ }
+ // Compute the cost of all instructions only feeding the exit conditions.
+ for (unsigned I = 0; I != ExitInstrs.size(); ++I) {
+ Instruction *CondI = ExitInstrs[I];
+ if (!OrigLoop->contains(CondI) ||
+ !CostCtx.SkipCostComputation.insert(CondI).second)
+ continue;
+ Cost += CostCtx.getLegacyCost(CondI, VF);
+ for (Value *Op : CondI->operands()) {
+ auto *OpI = dyn_cast<Instruction>(Op);
+ if (!OpI || any_of(OpI->users(), [&ExitInstrs](User *U) {
+ return !ExitInstrs.contains(cast<Instruction>(U));
+ }))
+ continue;
+ ExitInstrs.insert(OpI);
+ }
+ }
+
+ // The legacy cost model has special logic to compute the cost of in-loop
+ // reductions, which may be smaller than the sum of all instructions involved
+ // in the reduction. For AnyOf reductions, VPlan codegen may remove the select
+ // which the legacy cost model uses to assign cost. Pre-compute their costs
+ // for now.
+ // TODO: Switch to costing based on VPlan once the logic has been ported.
+ for (const auto &[RedPhi, RdxDesc] : Legal->getReductionVars()) {
+ if (!CM.isInLoopReduction(RedPhi) &&
+ !RecurrenceDescriptor::isAnyOfRecurrenceKind(
+ RdxDesc.getRecurrenceKind()))
+ continue;
+
+ // AnyOf reduction codegen may remove the select. To match the legacy cost
+ // model, pre-compute the cost for AnyOf reductions here.
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
+ RdxDesc.getRecurrenceKind())) {
+ auto *Select = cast<SelectInst>(*find_if(
+ RedPhi->users(), [](User *U) { return isa<SelectInst>(U); }));
+ assert(!CostCtx.SkipCostComputation.contains(Select) &&
+ "reduction op visited multiple times");
+ CostCtx.SkipCostComputation.insert(Select);
+ auto ReductionCost = CostCtx.getLegacyCost(Select, VF);
+ LLVM_DEBUG(dbgs() << "Cost of " << ReductionCost << " for VF " << VF
+ << ":\n any-of reduction " << *Select << "\n");
+ Cost += ReductionCost;
+ continue;
+ }
+
+ const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop);
+ SetVector<Instruction *> ChainOpsAndOperands(ChainOps.begin(),
+ ChainOps.end());
+ // Also include the operands of instructions in the chain, as the cost-model
+ // may mark extends as free.
+ for (auto *ChainOp : ChainOps) {
+ for (Value *Op : ChainOp->operands()) {
+ if (auto *I = dyn_cast<Instruction>(Op))
+ ChainOpsAndOperands.insert(I);
+ }
+ }
+
+ // Pre-compute the cost for I, if it has a reduction pattern cost.
+ for (Instruction *I : ChainOpsAndOperands) {
+ auto ReductionCost = CM.getReductionPatternCost(
+ I, VF, ToVectorTy(I->getType(), VF), TTI::TCK_RecipThroughput);
+ if (!ReductionCost)
+ continue;
+
+ assert(!CostCtx.SkipCostComputation.contains(I) &&
+ "reduction op visited multiple times");
+ CostCtx.SkipCostComputation.insert(I);
+ LLVM_DEBUG(dbgs() << "Cost of " << ReductionCost << " for VF " << VF
+ << ":\n in-loop reduction " << *I << "\n");
+ Cost += *ReductionCost;
+ }
+ }
+
+ // Pre-compute the costs for branches except for the backedge, as the number
+ // of replicate regions in a VPlan may not directly match the number of
+ // branches, which would lead to
diff erent decisions.
+ // TODO: Compute cost of branches for each replicate region in the VPlan,
+ // which is more accurate than the legacy cost model.
+ for (BasicBlock *BB : OrigLoop->blocks()) {
+ if (BB == OrigLoop->getLoopLatch())
+ continue;
+ CostCtx.SkipCostComputation.insert(BB->getTerminator());
+ auto BranchCost = CostCtx.getLegacyCost(BB->getTerminator(), VF);
+ Cost += BranchCost;
+ }
+ // Now compute and add the VPlan-based cost.
+ Cost += Plan.cost(VF, CostCtx);
+ LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost << "\n");
+ return Cost;
+}
+
+VPlan &LoopVectorizationPlanner::getBestPlan() const {
+ // If there is a single VPlan with a single VF, return it directly.
+ VPlan &FirstPlan = *VPlans[0];
+ if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
+ return FirstPlan;
+
+ VPlan *BestPlan = &FirstPlan;
+ ElementCount ScalarVF = ElementCount::getFixed(1);
+ assert(hasPlanWithVF(ScalarVF) &&
+ "More than a single plan/VF w/o any plan having scalar VF");
+
+ // TODO: Compute scalar cost using VPlan-based cost model.
+ InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
+ VectorizationFactor BestFactor(ScalarVF, ScalarCost, ScalarCost);
+
+ bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+ if (ForceVectorization) {
+ // Ignore scalar width, because the user explicitly wants vectorization.
+ // Initialize cost to max so that VF = 2 is, at least, chosen during cost
+ // evaluation.
+ BestFactor.Cost = InstructionCost::getMax();
+ }
+
+ for (auto &P : VPlans) {
+ for (ElementCount VF : P->vectorFactors()) {
+ if (VF.isScalar())
+ continue;
+ if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Not considering vector loop of width " << VF
+ << " because it will not generate any vector instructions.\n");
+ continue;
+ }
+
+ InstructionCost Cost = cost(*P, VF);
+ VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
+ if (isMoreProfitable(CurrentFactor, BestFactor)) {
+ BestFactor = CurrentFactor;
+ BestPlan = &*P;
+ }
+ }
+ }
+ BestPlan->setVF(BestFactor.Width);
+ return *BestPlan;
+}
+
VPlan &LoopVectorizationPlanner::getBestPlanFor(ElementCount VF) const {
assert(count_if(VPlans,
[VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
@@ -10169,8 +10369,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VF.MinProfitableTripCount, IC, &LVL, &CM, BFI,
PSI, Checks);
- VPlan &BestPlan = LVP.getBestPlanFor(VF.Width);
- LVP.executePlan(VF.Width, IC, BestPlan, LB, DT, false);
+ VPlan &BestPlan = LVP.getBestPlan();
+ assert(size(BestPlan.vectorFactors()) == 1 &&
+ "Plan should have a single VF");
+ ElementCount Width = *BestPlan.vectorFactors().begin();
+ LLVM_DEBUG(dbgs() << "VF picked by VPlan cost model: " << Width
+ << "\n");
+ assert(VF.Width == Width &&
+ "VPlan cost model and legacy cost model disagreed");
+ LVP.executePlan(Width, IC, BestPlan, LB, DT, false);
++LoopsVectorized;
// Add metadata to disable runtime unrolling a scalar loop when there
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 1018a5e8c5707..83a035fb4df86 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -776,6 +776,48 @@ void VPRegionBlock::execute(VPTransformState *State) {
State->Instance.reset();
}
+InstructionCost VPBasicBlock::cost(ElementCount VF, VPCostContext &Ctx) {
+ InstructionCost Cost = 0;
+ for (VPRecipeBase &R : Recipes)
+ Cost += R.cost(VF, Ctx);
+ return Cost;
+}
+
+InstructionCost VPRegionBlock::cost(ElementCount VF, VPCostContext &Ctx) {
+ if (!isReplicator()) {
+ InstructionCost Cost = 0;
+ for (VPBlockBase *Block : vp_depth_first_shallow(getEntry()))
+ Cost += Block->cost(VF, Ctx);
+ InstructionCost BackedgeCost =
+ Ctx.TTI.getCFInstrCost(Instruction::Br, TTI::TCK_RecipThroughput);
+ LLVM_DEBUG(dbgs() << "Cost of " << BackedgeCost << " for VF " << VF
+ << ": vector loop backedge\n");
+ Cost += BackedgeCost;
+ return Cost;
+ }
+
+ // Compute the cost of a replicate region. Replicating isn't supported for
+ // scalable vectors, return an invalid cost for them.
+ // TODO: Discard scalable VPlans with replicate recipes earlier after
+ // construction.
+ if (VF.isScalable())
+ return InstructionCost::getInvalid();
+
+ // First compute the cost of the conditionally executed recipes, followed by
+ // account for the branching cost, except if the mask is a header mask or
+ // uniform condition.
+ using namespace llvm::VPlanPatternMatch;
+ VPBasicBlock *Then = cast<VPBasicBlock>(getEntry()->getSuccessors()[0]);
+ InstructionCost ThenCost = Then->cost(VF, Ctx);
+
+ // For the scalar case, we may not always execute the original predicated
+ // block, Thus, scale the block's cost by the probability of executing it.
+ if (VF.isScalar())
+ return ThenCost / getReciprocalPredBlockProb();
+
+ return ThenCost;
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@@ -1040,6 +1082,12 @@ void VPlan::execute(VPTransformState *State) {
"DT not preserved correctly");
}
+InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) {
+ // For now only return the cost of the vector loop region, ignoring any other
+ // blocks, like the preheader or middle blocks.
+ return getVectorLoopRegion()->cost(VF, Ctx);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPlan::printLiveIns(raw_ostream &O) const {
VPSlotTracker SlotTracker(this);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f397c7a6e1036..223e3325b2d70 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -42,6 +42,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/FMF.h"
#include "llvm/IR/Operator.h"
+#include "llvm/Support/InstructionCost.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -64,8 +65,11 @@ class VPlan;
class VPReplicateRecipe;
class VPlanSlp;
class Value;
+class LoopVectorizationCostModel;
class LoopVersioning;
+struct VPCostContext;
+
namespace Intrinsic {
typedef unsigned ID;
}
@@ -82,6 +86,14 @@ Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
const SCEV *createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE,
Loop *CurLoop = nullptr);
+/// A helper function that returns the reciprocal of the block probability of
+/// predicated blocks. If we return X, we are assuming the predicated block
+/// will execute once for every X iterations of the loop header.
+///
+/// TODO: We should use actual block probability here, if available. Currently,
+/// we always assume predicated blocks have a 50% chance of executing.
+inline unsigned getReciprocalPredBlockProb() { return 2; }
+
/// A range of powers-of-2 vectorization factors with fixed start and
/// adjustable end. The range includes start and excludes end, e.g.,:
/// [1, 16) = {1, 2, 4, 8}
@@ -633,6 +645,9 @@ class VPBlockBase {
/// VPBlockBase, thereby "executing" the VPlan.
virtual void execute(VPTransformState *State) = 0;
+ /// Return the cost of the block.
+ virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx) = 0;
+
/// Delete all blocks reachable from a given VPBlockBase, inclusive.
static void deleteCFG(VPBlockBase *Entry);
@@ -716,6 +731,27 @@ class VPLiveOut : public VPUser {
#endif
};
+/// Struct to hold various analysis needed for cost computations.
+struct VPCostContext {
+ const TargetTransformInfo &TTI;
+ VPTypeAnalysis Types;
+ LLVMContext &LLVMCtx;
+ LoopVectorizationCostModel &CM;
+ SmallPtrSet<Instruction *, 8> SkipCostComputation;
+
+ VPCostContext(const TargetTransformInfo &TTI, Type *CanIVTy,
+ LLVMContext &LLVMCtx, LoopVectorizationCostModel &CM)
+ : TTI(TTI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
+
+ /// Return the cost for \p UI with \p VF using the legacy cost model as
+ /// fallback until computing the cost of all recipes migrates to VPlan.
+ InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const;
+
+ /// Return true if the cost for \p UI shouldn't be computed, e.g. because it
+ /// has already been pre-computed.
+ bool skipCostComputation(Instruction *UI, bool IsVector) const;
+};
+
/// VPRecipeBase is a base class modeling a sequence of one or more output IR
/// instructions. VPRecipeBase owns the VPValues it defines through VPDef
/// and is responsible for deleting its defined values. Single-value
@@ -755,6 +791,11 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
/// this VPRecipe, thereby "executing" the VPlan.
virtual void execute(VPTransformState &State) = 0;
+ /// Return the cost of this recipe, taking into account if the cost
+ /// computation should be skipped and the ForceTargetInstructionCost flag.
+ /// Also takes care of printing the cost for debugging.
+ virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
+
/// Insert an unlinked recipe into a basic block immediately before
/// the specified recipe.
void insertBefore(VPRecipeBase *InsertPos);
@@ -815,6 +856,11 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
/// Returns the debug location of the recipe.
DebugLoc getDebugLoc() const { return DL; }
+
+protected:
+ /// Compute the cost of this recipe using the legacy cost model and the
+ /// underlying instructions.
+ InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
};
// Helper macro to define common classof implementations for recipes.
@@ -1390,8 +1436,6 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags {
ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
- assert(UI.getType() == ResultTy &&
- "result type of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
@@ -2105,6 +2149,8 @@ class VPInterleaveRecipe : public VPRecipeBase {
"Op must be an operand of the recipe");
return Op == getAddr() && !llvm::is_contained(getStoredValues(), Op);
}
+
+ Instruction *getInsertPos() const { return IG->getInsertPos(); }
};
/// A recipe to represent inloop reduction operations, performing a reduction on
@@ -2919,6 +2965,9 @@ class VPBasicBlock : public VPBlockBase {
/// this VPBasicBlock, thereby "executing" the VPlan.
void execute(VPTransformState *State) override;
+ /// Return the cost of this VPBasicBlock.
+ InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
+
/// Return the position of the first non-phi node recipe in the block.
iterator getFirstNonPhi();
@@ -3093,6 +3142,9 @@ class VPRegionBlock : public VPBlockBase {
/// this VPRegionBlock, thereby "executing" the VPlan.
void execute(VPTransformState *State) override;
+ // Return the cost of this region.
+ InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
+
void dropAllReferences(VPValue *NewValue) override;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -3216,6 +3268,9 @@ class VPlan {
/// Generate the IR code for this VPlan.
void execute(VPTransformState *State);
+ /// Return the cost of this plan.
+ InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
+
VPBasicBlock *getEntry() { return Entry; }
const VPBasicBlock *getEntry() const { return Entry; }
@@ -3687,7 +3742,6 @@ inline bool isUniformAfterVectorization(VPValue *VPV) {
/// Return true if \p V is a header mask in \p Plan.
bool isHeaderMask(VPValue *V, VPlan &Plan);
-
} // end namespace vputils
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d315b637464e9..b0fff09b0d102 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -39,6 +39,7 @@ using VectorParts = SmallVector<Value *, 2>;
namespace llvm {
extern cl::opt<bool> EnableVPlanNativePath;
}
+extern cl::opt<unsigned> ForceTargetInstructionCost;
#define LV_NAME "loop-vectorize"
#define DEBUG_TYPE LV_NAME
@@ -253,6 +254,49 @@ void VPRecipeBase::moveBefore(VPBasicBlock &BB,
insertBefore(BB, I);
}
+/// Return the underlying instruction to be used for computing \p R's cost via
+/// the legacy cost model. Return nullptr if there's no suitable instruction.
+static Instruction *getInstructionForCost(const VPRecipeBase *R) {
+ if (auto *S = dyn_cast<VPSingleDefRecipe>(R))
+ return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
+ if (auto *IG = dyn_cast<VPInterleaveRecipe>(R))
+ return IG->getInsertPos();
+ if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
+ return &WidenMem->getIngredient();
+ return nullptr;
+}
+
+InstructionCost VPRecipeBase::cost(ElementCount VF, VPCostContext &Ctx) {
+ if (auto *UI = getInstructionForCost(this))
+ if (Ctx.skipCostComputation(UI, VF.isVector()))
+ return 0;
+
+ InstructionCost RecipeCost = computeCost(VF, Ctx);
+ if (ForceTargetInstructionCost.getNumOccurrences() > 0 &&
+ RecipeCost.isValid())
+ RecipeCost = InstructionCost(ForceTargetInstructionCost);
+
+ LLVM_DEBUG({
+ dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": ";
+ dump();
+ });
+ return RecipeCost;
+}
+
+InstructionCost VPRecipeBase::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ // Compute the cost for the recipe falling back to the legacy cost model using
+ // the underlying instruction. If there is no underlying instruction, returns
+ // 0.
+ Instruction *UI = getInstructionForCost(this);
+ if (UI && isa<VPReplicateRecipe>(this)) {
+ // VPReplicateRecipe may be cloned as part of an existing VPlan-to-VPlan
+ // transform, avoid computing their cost multiple times for now.
+ Ctx.SkipCostComputation.insert(UI);
+ }
+ return UI ? Ctx.getLegacyCost(UI, VF) : 0;
+}
+
FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const {
assert(OpType == OperationType::FPMathOp &&
"recipe doesn't have fast math flags");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fc213320d5a35..d502a92bcd6e6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1020,6 +1020,10 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
: Instruction::ZExt;
auto *VPC =
new VPWidenCastRecipe(Instruction::CastOps(ExtOpcode), A, TruncTy);
+ if (auto *UnderlyingExt = R.getOperand(0)->getUnderlyingValue()) {
+ // UnderlyingExt has distinct return type, used to retain legacy cost.
+ VPC->setUnderlyingValue(UnderlyingExt);
+ }
VPC->insertBefore(&R);
Trunc->replaceAllUsesWith(VPC);
} else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
@@ -1536,6 +1540,7 @@ void VPlanTransforms::dropPoisonGeneratingRecipes(
VPInstruction *New = Builder.createOverflowingOp(
Instruction::Add, {A, B}, {false, false},
RecWithFlags->getDebugLoc());
+ New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
RecWithFlags->replaceAllUsesWith(New);
RecWithFlags->eraseFromParent();
CurRec = New;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8d945f6f2b8ea..fa6a65ff2f3ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -74,8 +74,7 @@ class VPValue {
public:
/// Return the underlying Value attached to this VPValue.
- Value *getUnderlyingValue() { return UnderlyingVal; }
- const Value *getUnderlyingValue() const { return UnderlyingVal; }
+ Value *getUnderlyingValue() const { return UnderlyingVal; }
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 01e69b4722c70..8e9713fecf29d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -132,6 +132,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Interleaving is not beneficial.
; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in <stdin>
; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop
+; CHECK-NEXT: VF picked by VPlan cost model: vscale x 4
; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1
; CHECK: LV: Interleaving disabled by the pass manager
; CHECK-NEXT: LV: Loop does not require scalar epilogue
@@ -287,6 +288,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Interleaving is not beneficial.
; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in <stdin>
; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop
+; CHECK-NEXT: VF picked by VPlan cost model: vscale x 4
; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1
; CHECK: LV: Interleaving disabled by the pass manager
; CHECK-NEXT: LV: Loop does not require scalar epilogue
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/x86-loopvectorize-costmodel.ll.expected b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/x86-loopvectorize-costmodel.ll.expected
index e862bf87d265c..5aa270e76f4c8 100644
--- a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/x86-loopvectorize-costmodel.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/x86-loopvectorize-costmodel.ll.expected
@@ -17,6 +17,7 @@ define void @test() {
; CHECK: LV: Found an estimated cost of 5 for VF 16 For instruction: %v0 = load float, ptr %in0, align 4
; CHECK: LV: Found an estimated cost of 22 for VF 32 For instruction: %v0 = load float, ptr %in0, align 4
; CHECK: LV: Found an estimated cost of 92 for VF 64 For instruction: %v0 = load float, ptr %in0, align 4
+; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %v0 = load float, ptr %in0, align 4
;
entry:
br label %for.body
More information about the llvm-commits
mailing list