[llvm] 0c62d83 - [VPlan] Record estimated branch probabilities on VPlan0 for cost modeling (#216172)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 02:28:41 PDT 2026
Author: Florian Hahn
Date: 2026-09-09T10:28:36+01:00
New Revision: 0c62d834abff9b180def1b32d3cf9c225d504994
URL: https://github.com/llvm/llvm-project/commit/0c62d834abff9b180def1b32d3cf9c225d504994
DIFF: https://github.com/llvm/llvm-project/commit/0c62d834abff9b180def1b32d3cf9c225d504994.diff
LOG: [VPlan] Record estimated branch probabilities on VPlan0 for cost modeling (#216172)
Update execution frequency/probability tracking in VPlan to also include
estimates from BFI, and use the information in VPlan for VPlan-based
cost computations, removing another remaining IR-based lookup.
We need to differentiate between estimated and concrete profile
information when annotating branches with information from BFI (using
custom !vplan.prof.estimated metadata instead of !prof) and track if a
computed frequency contains any estimates using an extra bool which is
also encoded in the metadata (so we do not synthesize !prof
for such frequencies during codegen).
The new code should only request BFI if needed, and compile-time impact
should be in the noise:
https://llvm-compile-time-tracker.com/compare.php?from=8436dd4a2bfe57d0b3fb6c9c11c69d75c1d1bccb&to=4d60694fed5e5765d4f5a2a72aac7b8674e512e2&stat=instructions:u
PR: https://github.com/llvm/llvm-project/pull/216172
Added:
Modified:
llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanHelpers.h
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.h
llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index 5eba1a0f8527c..73c607ad1cf5a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -66,6 +66,7 @@ namespace llvm {
class AssumptionCache;
class BlockFrequencyInfo;
+class BranchProbabilityInfo;
class DemandedBits;
class DominatorTree;
class Function;
@@ -146,6 +147,7 @@ struct LoopVectorizePass : public OptionalPassInfoMixin<LoopVectorizePass> {
TargetTransformInfo *TTI;
DominatorTree *DT;
std::function<BlockFrequencyInfo &()> GetBFI;
+ std::function<const BranchProbabilityInfo &()> GetBPI;
TargetLibraryInfo *TLI;
DemandedBits *DB;
AssumptionCache *AC;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 4ceac108ae8c7..47311977dd379 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -36,6 +36,7 @@ class GeneratedRTChecks;
namespace llvm {
+class BranchProbabilityInfo;
class LoopInfo;
class DominatorTree;
class LoopVectorizationLegality;
@@ -892,6 +893,9 @@ class LoopVectorizationPlanner {
OptimizationRemarkEmitter *ORE;
+ /// Lazily fetch BranchProbabilityInfo, independent of BlockFrequencyInfo.
+ std::function<const BranchProbabilityInfo &()> GetBPI;
+
SmallVector<VPlanPtr, 4> VPlans;
/// Profitable vector factors.
@@ -922,7 +926,8 @@ class LoopVectorizationPlanner {
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
std::unique_ptr<LoopVectorizationCostModel> CM,
VFSelectionContext &Config, InterleavedAccessInfo &IAI,
- PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
+ PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE,
+ std::function<const BranchProbabilityInfo &()> GetBPI);
~LoopVectorizationPlanner();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2493d2aca6265..82b1faaaa9b23 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -80,6 +80,7 @@
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/CycleAnalysis.h"
@@ -5347,10 +5348,6 @@ void VPCostContext::invalidateWideningDecision(Instruction *I,
LoopVectorizationCostModel::CM_InvalidatedDecision, 0);
}
-uint64_t VPCostContext::getPredBlockCostDivisor(BasicBlock *BB) const {
- return CM.getPredBlockCostDivisor(CostKind, BB);
-}
-
bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
return CM.isScalarWithPredication(I, VF) ||
CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
@@ -5669,9 +5666,11 @@ LoopVectorizationPlanner::LoopVectorizationPlanner(
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- OptimizationRemarkEmitter *ORE)
+ OptimizationRemarkEmitter *ORE,
+ std::function<const BranchProbabilityInfo &()> GetBPI)
: OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
+ CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE),
+ GetBPI(GetBPI) {}
LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
@@ -6330,16 +6329,16 @@ static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
Edges += VPBB->getNumSuccessors();
uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
- DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies =
- vputils::computeExecutionFrequencies(Blocks);
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies = vputils::computeExecutionFrequencies(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
// Compare at BranchProbability's coarser resolution, which is as precise as
// BFI's frequencies get.
- std::optional<BlockFrequency> Freq = Frequencies.lookup(VPBB);
+ std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
if (!Freq)
continue;
- BranchProbability Computed = vputils::getExecutionProbability(*Freq);
+ BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
// Clamp to the header's frequency, which BFI's rounding may exceed.
uint64_t BBFreq = BFI.getBlockFreq(BB).getFrequency();
@@ -6378,9 +6377,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// Create initial base VPlan0, to serve as common starting point for all
// candidates built later for specific VF ranges.
- auto VPlan0 = VPlanTransforms::buildVPlan0(OrigLoop, *LI,
- Legal->getWidestInductionType(),
- PSE, LVer ? &*LVer : nullptr);
+ auto VPlan0 = VPlanTransforms::buildVPlan0(
+ OrigLoop, *LI, Legal->getWidestInductionType(), PSE,
+ LVer ? &*LVer : nullptr, GetBPI);
VPDominatorTree VPDT(*VPlan0);
if (const LoopAccessInfo *LAI = Legal->getLAI())
@@ -7928,7 +7927,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
L, LI, DT, TLI, *TTI, &LVL,
std::make_unique<LoopVectorizationCostModel>(
SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
- Config, IAI, PSE, ORE);
+ Config, IAI, PSE, ORE, GetBPI);
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
@@ -8338,12 +8337,21 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
- GetBFI = [this, &AM, &F]() -> BlockFrequencyInfo & {
- // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
+ // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
+ // Both BlockFrequencyAnalysis and BranchProbabilityAnalysis depend on it, so
+ // drop the stale result before either is (re-)computed.
+ auto ClearStaleCycleInfo = [this, &AM, &F] {
if (CFGChanged && AM.getCachedResult<CycleAnalysis>(F))
AM.clearAnalysis<CycleAnalysis>(F);
+ };
+ GetBFI = [&AM, &F, ClearStaleCycleInfo]() -> BlockFrequencyInfo & {
+ ClearStaleCycleInfo();
return AM.getResult<BlockFrequencyAnalysis>(F);
};
+ GetBPI = [&AM, &F, ClearStaleCycleInfo]() -> const BranchProbabilityInfo & {
+ ClearStaleCycleInfo();
+ return AM.getResult<BranchProbabilityAnalysis>(F);
+ };
LoopVectorizeResult Result = runImpl(F);
if (!Result.MadeAnyChange)
return PreservedAnalyses::all();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 8b42ccb6803c8..812f46cee9dbb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -2000,3 +2000,18 @@ bool VPCostContext::isFreeScalarIntrinsic(Intrinsic::ID ID) {
Intrinsic::experimental_noalias_scope_decl},
ID);
}
+
+uint64_t VPCostContext::getReplicateRegionCostDivisor(
+ const VPRegionBlock *Region) const {
+ if (CostKind == TTI::TCK_CodeSize)
+ return 1;
+ std::optional<VPExecutionFrequency> Freq =
+ Region->getEntryBranchOnMask()->getExecutionFrequency();
+ if (!Freq)
+ return 1;
+ // A recorded frequency is neither zero nor always-executing, so the
+ // probability is non-zero and the division below is safe.
+ return divideNearest(
+ BranchProbability::getDenominator(),
+ vputils::getExecutionProbability(Freq->Freq).getNumerator());
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a9d292314bde7..2393403b3a837 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1177,6 +1177,17 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
VPCostContext &Ctx) const;
};
+/// The frequency with which a recipe executes, relative to the entry of the
+/// loop region. IsEstimated is set if any branch weight it was composed from
+/// was estimated from static heuristics.
+struct VPExecutionFrequency {
+ const BlockFrequency Freq;
+ const bool IsEstimated;
+
+ VPExecutionFrequency(BlockFrequency Freq, bool IsEstimated)
+ : Freq(Freq), IsEstimated(IsEstimated) {}
+};
+
/// Helper to manage IR metadata for recipes. It filters out metadata that
/// cannot be propagated.
class LLVM_ABI_FOR_TEST VPIRMetadata {
@@ -1186,6 +1197,10 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
static constexpr StringLiteral ExecutionFrequencyMDName =
"vplan.execution.frequency";
+ /// Name of the VPlan-internal metadata kind holding estimated branch weights.
+ static constexpr StringLiteral EstimatedProfileMDName =
+ "vplan.prof.estimated";
+
/// Returns the ID of the metadata kind named \p Kind, taking the context from
/// any attached node; all belong to the context of the VPlan's function.
unsigned getMDKindID(StringRef Kind) const {
@@ -1193,6 +1208,12 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
return Metadata.front().second->getContext().getMDKindID(Kind);
}
+ /// Returns the node attached under the VPlan-internal metadata kind named
+ /// \p Kind, or nullptr if there is none.
+ MDNode *getInternalMetadata(StringRef Kind) const {
+ return Metadata.empty() ? nullptr : getMetadata(getMDKindID(Kind));
+ }
+
public:
VPIRMetadata() = default;
@@ -1240,16 +1261,35 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
}
/// Record that the recipe executes with frequency \p Freq, relative to the
- /// entry of the loop region; see vputils::AlwaysExecutesFreq.
- void setExecutionFrequency(std::optional<BlockFrequency> Freq,
+ /// entry of the loop region.
+ void setExecutionFrequency(std::optional<VPExecutionFrequency> Freq,
LLVMContext &Ctx);
/// Returns the frequency recorded by setExecutionFrequency, if any.
- std::optional<BlockFrequency> getExecutionFrequency() const;
+ std::optional<VPExecutionFrequency> getExecutionFrequency() const;
/// Drop the frequency recorded by setExecutionFrequency, if any.
void clearExecutionFrequency();
+ /// Returns the branch weights recorded for this terminator, preferring real
+ /// profile data over an estimate, or nullptr if there are none.
+ MDNode *getBranchWeights() const {
+ MDNode *Node = getMetadata(LLVMContext::MD_prof);
+ return Node ? Node : getInternalMetadata(EstimatedProfileMDName);
+ }
+
+ /// Returns true if the weights returned by getBranchWeights are estimated.
+ bool hasEstimatedBranchWeights() const {
+ return getInternalMetadata(EstimatedProfileMDName);
+ }
+
+ /// Set estimated branch weights to \p Node.
+ void setEstimatedBranchWeights(MDNode *Node) {
+ assert(!getMetadata(LLVMContext::MD_prof) &&
+ "real profile data takes precedence over an estimate");
+ setMetadata(Node->getContext().getMDKindID(EstimatedProfileMDName), Node);
+ }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print metadata with node IDs.
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c0c63f3e87842..17a3b9790aadb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -20,7 +20,9 @@
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
+#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVectorExtras.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
@@ -53,6 +55,12 @@ class PlainCFGBuilder {
// Loop versioning for alias metadata.
LoopVersioning *LVer;
+ // Lazily provides branch probabilities for the incoming IR.
+ function_ref<const BranchProbabilityInfo &()> GetBPI;
+
+ // The BranchProbabilityInfo returned by GetBPI, cached on first use.
+ const BranchProbabilityInfo *BPI = nullptr;
+
// Vectorization plan that we are working on.
std::unique_ptr<VPlan> Plan;
@@ -79,10 +87,12 @@ class PlainCFGBuilder {
#endif
VPValue *getOrCreateVPOperand(Value *IRVal);
void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
+ VPIRMetadata getTerminatorMetadata(Instruction &Term);
public:
- PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer, Type *IdxTy)
- : TheLoop(Lp), LI(LI), LVer(LVer),
+ PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer, Type *IdxTy,
+ function_ref<const BranchProbabilityInfo &()> GetBPI)
+ : TheLoop(Lp), LI(LI), LVer(LVer), GetBPI(GetBPI),
Plan(std::make_unique<VPlan>(Lp, IdxTy)) {}
/// Build plain CFG for TheLoop and connect it to Plan's entry.
@@ -183,6 +193,26 @@ VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
return NewVPVal;
}
+// Returns the metadata to preserve for terminator \p Term.
+VPIRMetadata PlainCFGBuilder::getTerminatorMetadata(Instruction &Term) {
+ VPIRMetadata MD(Term);
+ if (MD.getMetadata(LLVMContext::MD_prof))
+ return MD;
+ // Estimates are only read for edges inside the loop region.
+ if (!TheLoop->isInnermost() || Term.getParent() == TheLoop->getLoopLatch())
+ return MD;
+ // The weights describe the edges leaving Term in the order of its successors,
+ // matching the successor order of the VPBasicBlock created for Term's parent.
+ if (!BPI)
+ BPI = &GetBPI();
+ auto Weights = map_to_vector(seq(Term.getNumSuccessors()), [&](unsigned I) {
+ return BPI->getEdgeProbability(Term.getParent(), I).getNumerator();
+ });
+ MD.setEstimatedBranchWeights(
+ MDBuilder(Plan->getContext()).createBranchWeights(Weights));
+ return MD;
+}
+
// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
// counterpart. This function must be invoked in RPO so that the operands of a
// VPInstruction in \p BB have been visited before (except for Phi nodes).
@@ -207,7 +237,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// recipes.
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, {},
- VPIRMetadata(*Inst), Inst->getDebugLoc());
+ getTerminatorMetadata(*Inst),
+ Inst->getDebugLoc());
continue;
}
@@ -219,7 +250,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
for (auto Case : SI->cases())
Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, {},
- VPIRMetadata(*Inst), Inst->getDebugLoc());
+ getTerminatorMetadata(*Inst),
+ Inst->getDebugLoc());
continue;
}
@@ -598,11 +630,11 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
/// To make RUN_VPLAN_PASS print initial VPlan.
static void printAfterInitialConstruction(VPlan &) {}
-std::unique_ptr<VPlan>
-VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
- PredicatedScalarEvolution &PSE,
- LoopVersioning *LVer) {
- PlainCFGBuilder Builder(TheLoop, &LI, LVer, InductionTy);
+std::unique_ptr<VPlan> VPlanTransforms::buildVPlan0(
+ Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
+ PredicatedScalarEvolution &PSE, LoopVersioning *LVer,
+ function_ref<const BranchProbabilityInfo &()> GetBPI) {
+ PlainCFGBuilder Builder(TheLoop, &LI, LVer, InductionTy, GetBPI);
std::unique_ptr<VPlan> VPlan0 = Builder.buildPlainCFG();
addInitialSkeleton(*VPlan0, InductionTy, PSE, TheLoop);
simplifyLiveInsWithSCEV(*VPlan0, PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 2838a9e4895a6..d31c5acdf69a9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -356,9 +356,9 @@ struct VPCostContext {
/// transform replaced the original recipe.
void invalidateWideningDecision(Instruction *I, ElementCount VF);
- /// \returns how much the cost of a predicated block should be divided by.
- /// Forwards to LoopVectorizationCostModel::getPredBlockCostDivisor.
- uint64_t getPredBlockCostDivisor(BasicBlock *BB) const;
+ /// \returns how much the cost of the block predicated by replicate region
+ /// \p Region should be divided by.
+ uint64_t getReplicateRegionCostDivisor(const VPRegionBlock *Region) const;
/// Returns true if \p I is known to be scalarized at \p VF.
bool willBeScalarized(Instruction *I, ElementCount VF) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index cbc41e82fa87c..e056b2b613b33 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -406,8 +406,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Header);
// Non-outer regions with VPBBs only are supported at the moment.
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies =
- vputils::computeExecutionFrequencies(Blocks);
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies = vputils::computeExecutionFrequencies(Blocks);
VPPredicator Predicator(Plan);
for (VPBasicBlock *VPBB : Blocks) {
@@ -423,7 +423,7 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
// Mask all VPInstructions in the block and record the frequency with
// which the masked recipes execute.
- std::optional<BlockFrequency> Freq = Frequencies.lookup(VPBB);
+ std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1a6119b22c9d3..fce56b42de84c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2090,41 +2090,43 @@ void VPIRPhi::printRecipe(raw_ostream &O, const Twine &Indent,
void VPIRMetadata::applyMetadata(Instruction &I) const {
if (Metadata.empty())
return;
- // The execution frequency is VPlan-internal and must not reach IR.
+ // Frequencies and estimated branch weights are VPlan-internal and must not
+ // reach IR.
unsigned ExecFreqKind = getMDKindID(ExecutionFrequencyMDName);
+ unsigned EstProfKind = getMDKindID(EstimatedProfileMDName);
for (const auto &[Kind, Node] : Metadata)
- if (Kind != ExecFreqKind)
+ if (Kind != ExecFreqKind && Kind != EstProfKind)
I.setMetadata(Kind, Node);
}
/// Returns the execution frequency recorded in \p Node.
-static BlockFrequency getExecutionFrequencyFromMD(const MDNode *Node) {
+static VPExecutionFrequency getExecutionFrequencyFromMD(const MDNode *Node) {
+ assert(Node->getNumOperands() <= 2 && "unexpected frequency node shape");
uint64_t Freq =
mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue();
assert(Freq <= vputils::AlwaysExecutesFreq &&
"frequency cannot exceed the one of an always executing block");
- return BlockFrequency(Freq);
+ return {BlockFrequency(Freq), Node->getNumOperands() == 2};
}
-void VPIRMetadata::setExecutionFrequency(std::optional<BlockFrequency> Freq,
- LLVMContext &Ctx) {
+void VPIRMetadata::setExecutionFrequency(
+ std::optional<VPExecutionFrequency> Freq, LLVMContext &Ctx) {
// A recipe that never or always executes needs no annotation.
- if (!Freq || Freq->getFrequency() == 0 ||
- Freq->getFrequency() == vputils::AlwaysExecutesFreq)
+ if (!Freq || Freq->Freq.getFrequency() == 0 ||
+ Freq->Freq.getFrequency() == vputils::AlwaysExecutesFreq)
return;
- Constant *Frequency =
- ConstantInt::get(Type::getInt64Ty(Ctx), Freq->getFrequency());
- setMetadata(Ctx.getMDKindID(ExecutionFrequencyMDName),
- MDNode::get(Ctx, {ConstantAsMetadata::get(Frequency)}));
+ SmallVector<llvm::Metadata *, 2> Ops = {ConstantAsMetadata::get(
+ ConstantInt::get(Type::getInt64Ty(Ctx), Freq->Freq.getFrequency()))};
+ if (Freq->IsEstimated)
+ Ops.push_back(ConstantAsMetadata::get(ConstantInt::getTrue(Ctx)));
+ setMetadata(Ctx.getMDKindID(ExecutionFrequencyMDName), MDNode::get(Ctx, Ops));
}
-std::optional<BlockFrequency> VPIRMetadata::getExecutionFrequency() const {
- if (Metadata.empty())
- return std::nullopt;
- MDNode *Node = getMetadata(getMDKindID(ExecutionFrequencyMDName));
- if (!Node)
- return std::nullopt;
- return getExecutionFrequencyFromMD(Node);
+std::optional<VPExecutionFrequency>
+VPIRMetadata::getExecutionFrequency() const {
+ if (MDNode *Node = getInternalMetadata(ExecutionFrequencyMDName))
+ return getExecutionFrequencyFromMD(Node);
+ return std::nullopt;
}
void VPIRMetadata::clearExecutionFrequency() {
@@ -2163,15 +2165,21 @@ void VPIRMetadata::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
// Print the values of branch weights, which are more informative than the
// ID of the metadata node holding them.
SmallVector<uint32_t> Weights;
- if (Kind == LLVMContext::MD_prof && extractBranchWeights(Node, Weights)) {
+ bool IsEstimatedProfile = MDNames[Kind] == EstimatedProfileMDName;
+ if ((Kind == LLVMContext::MD_prof || IsEstimatedProfile) &&
+ extractBranchWeights(Node, Weights)) {
+ if (IsEstimatedProfile)
+ O << "estimated ";
O << "{";
interleaveComma(Weights, O);
O << "}";
} else if (MDNames[Kind] == ExecutionFrequencyMDName) {
// Print the frequency together with the probability it corresponds to.
- uint64_t Freq = getExecutionFrequencyFromMD(Node).getFrequency();
- O << Freq
- << format(" (%.4g%%)", 100.0 * Freq / vputils::AlwaysExecutesFreq);
+ auto [Freq, IsEstimated] = getExecutionFrequencyFromMD(Node);
+ O << Freq.getFrequency()
+ << format(" (%.4g%%%s)",
+ 100.0 * Freq.getFrequency() / vputils::AlwaysExecutesFreq,
+ IsEstimated ? ", estimated" : "");
} else {
Node->printAsOperand(O, M);
}
@@ -4016,7 +4024,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
// Scale the cost by the probability of executing the predicated blocks.
// This assumes the predicated block for each vector lane is equally
// likely.
- ScalarCost /= Ctx.getPredBlockCostDivisor(UI->getParent());
+ ScalarCost /= Ctx.getReplicateRegionCostDivisor(getRegion());
return ScalarCost;
}
case Instruction::Load:
@@ -4075,7 +4083,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
if (ParentRegion && ParentRegion->isReplicator()) {
if (!PtrSCEV)
break;
- Cost /= Ctx.getPredBlockCostDivisor(UI->getParent());
+ Cost /= Ctx.getReplicateRegionCostDivisor(ParentRegion);
Cost += Ctx.TTI.getCFInstrCost(Instruction::CondBr, Ctx.CostKind);
auto *VecI1Ty = VectorType::get(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 42ee24f3f5014..d12aec83f2856 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -496,12 +496,16 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
// If only one of the two is known, the higher one is unknown, so the
// result must be unknown too.
VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
- std::optional<BlockFrequency> Freq1 =
+ std::optional<VPExecutionFrequency> Freq1 =
Region1->getEntryBranchOnMask()->getExecutionFrequency();
- std::optional<BlockFrequency> Freq2 = Guard2->getExecutionFrequency();
+ std::optional<VPExecutionFrequency> Freq2 = Guard2->getExecutionFrequency();
if (Freq1 && Freq2) {
- if (*Freq2 < *Freq1)
+ if (Freq2->Freq < Freq1->Freq) {
+ // Freq1's frequency is taken, but it is only as trustworthy as the
+ // less trustworthy of the two.
+ Freq1.emplace(Freq1->Freq, Freq1->IsEstimated || Freq2->IsEstimated);
Guard2->setExecutionFrequency(Freq1, Plan.getContext());
+ }
} else if (Freq2) {
Guard2->clearExecutionFrequency();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6686d887bcf83..1c7b17942b795 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -24,6 +24,7 @@
namespace llvm {
+class BranchProbabilityInfo;
class InductionDescriptor;
class Instruction;
class Loop;
@@ -156,7 +157,8 @@ struct VPlanTransforms {
/// >[ ] <-- original loop exit block(s), wrapped in VPIRBasicBlocks.
LLVM_ABI_FOR_TEST static std::unique_ptr<VPlan>
buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
- PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr);
+ PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr,
+ function_ref<const BranchProbabilityInfo &()> GetBPI = nullptr);
/// Replace VPPhi recipes in \p Plan's header with corresponding
/// VPHeaderPHIRecipe subclasses for inductions, reductions, and
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index dea1b0db70d2f..52a16debc7acf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -676,13 +676,14 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
}
/// Converts the frequency \p Freq with which a block is entered to branch
-/// weights for the branch guarding it, or nullptr if \p Freq is unknown.
+/// weights for the branch guarding it, or nullptr if \p Freq is unknown or
+/// estimated.
static MDNode *
-convertFrequencyToBranchWeights(std::optional<BlockFrequency> Freq,
+convertFrequencyToBranchWeights(std::optional<VPExecutionFrequency> Freq,
LLVMContext &Ctx) {
- if (!Freq)
+ if (!Freq || Freq->IsEstimated)
return nullptr;
- BranchProbability P = vputils::getExecutionProbability(*Freq);
+ BranchProbability P = vputils::getExecutionProbability(Freq->Freq);
// Use the numerators of P and its complement as weights and reduce them via
// gcd to keep them small. Neither is zero, as P is neither zero nor one.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index befc2d477aac4..8327b30c7583e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1192,8 +1192,7 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
// below still visits each of them.
SmallVector<uint32_t> Weights;
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
- if (!Term ||
- !extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) ||
+ if (!Term || !extractBranchWeights(Term->getBranchWeights(), Weights) ||
Weights.size() != Successors.size())
Weights.assign(Successors.size(), 0);
uint64_t Total = sum_of(Weights, uint64_t(0));
@@ -1223,31 +1222,39 @@ static BlockFrequency scaleKeepingNonZero(BlockFrequency Freq,
return Scaled;
}
-DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>>
+DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
assert(!Blocks.empty() && "expected at least the header block");
// Push each block's frequency along its outgoing edges. Blocks is a DAG in
// reverse post-order (the loop region's backedge is implicit), so a block's
// frequency is final by the time it is visited.
- DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies;
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies;
Frequencies.reserve(Blocks.size());
// The header (first block) always executes, the others start out unreachable.
- Frequencies[Blocks.front()] = BlockFrequency(AlwaysExecutesFreq);
+ Frequencies[Blocks.front()].emplace(BlockFrequency(AlwaysExecutesFreq),
+ false);
for (VPBasicBlock *VPBB : Blocks.drop_front())
- Frequencies[VPBB] = BlockFrequency();
+ Frequencies[VPBB].emplace(BlockFrequency(), false);
for (VPBasicBlock *VPBB : Blocks) {
- std::optional<BlockFrequency> SrcFreq = Frequencies.at(VPBB);
+ std::optional<VPExecutionFrequency> Src = Frequencies.at(VPBB);
+ auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
+ bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- std::optional<BlockFrequency> &SuccFreq = Frequencies.at(Succ);
+ std::optional<VPExecutionFrequency> &SuccFreq = Frequencies.at(Succ);
// An unknown edge or predecessor poisons the successor.
- if (!SrcFreq || EdgeProb.isUnknown() || !SuccFreq) {
+ if (!Src || EdgeProb.isUnknown() || !SuccFreq) {
SuccFreq = std::nullopt;
continue;
}
// The sum can only exceed AlwaysExecutesFreq by rounding.
- SuccFreq = std::min(BlockFrequency(AlwaysExecutesFreq),
- *SuccFreq + scaleKeepingNonZero(*SrcFreq, EdgeProb));
+ BlockFrequency NewFreq =
+ std::min(BlockFrequency(AlwaysExecutesFreq),
+ SuccFreq->Freq + scaleKeepingNonZero(Src->Freq, EdgeProb));
+ bool NewIsEstimated =
+ SuccFreq->IsEstimated || Src->IsEstimated || TermIsEstimated;
+ SuccFreq.emplace(NewFreq, NewIsEstimated);
}
}
return Frequencies;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index cb9d1e5607a4a..738a5bc8b8066 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -232,10 +232,11 @@ inline constexpr uint64_t AlwaysExecutesFreq = 1ULL << 63;
BranchProbability getExecutionProbability(BlockFrequency Freq);
/// Computes for each block in \p Blocks, which must be in reverse post-order,
-/// the frequency with which it executes relative to the first (header) block.
+/// the frequency with which it executes relative to the first (header) block,
+/// and whether that frequency was composed using any estimated branch weights.
/// The frequency of a block is the sum over its incoming edges, or std::nullopt
/// if any edge on a path reaching it lacks branch weights.
-DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>>
+DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks);
namespace detail {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
index 0b0a698b42bf2..244c9add3b2dd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
@@ -223,14 +223,15 @@ exit:
}
define void @second_branch_without_weights(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
-; %merge's branch has no weights, so everything it reaches is unknown and stays
-; unannotated. %merge itself is still known, as both of its incoming edges are.
+; %merge's branch has no weights, so BranchProbabilityInfo estimates them.
+; %if.then.2's frequency is composed through that estimated edge, so it is
+; marked estimated too.
;
; %loop 1000/1000 = 1
; %if.then.1 250/1000 = 1/4
; %merge 1000/1000 = 1
-; %if.then.2 unknown
-; %latch unknown
+; %if.then.2 625/1000 = 5/8 (estimated)
+; %latch 1000/1000 = 1
;
; BFI-LABEL: block-frequency-info: second_branch_without_weights
; BFI-NEXT: - entry: float = 1.0,
@@ -252,7 +253,7 @@ define void @second_branch_without_weights(ptr noalias %a, ptr noalias %b, ptr n
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then.2:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0>{{$}}
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; VPLAN-NEXT: Successor(s): latch
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
index 1928c2c3e995a..8b4477346b55b 100644
--- a/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
@@ -72,8 +72,9 @@ exit:
}
; Without profile data the cost model is still self-contained: the recorded
-; reciprocal comes from block frequency info, which defaults to an even 50/50
-; split for the unprofiled predicate (reciprocal 2), so cost 10 / 2 -> 5.
+; reciprocal comes from branch weights estimated by BranchProbabilityInfo. For
+; this comparison against zero that heuristic favors the predicate at 5/8
+; (round(1/0.625) = 2), so cost 10 / 2 -> 5.
define void @predicated_udiv_no_profile(ptr %a, i32 %n) {
; CHECK-LABEL: 'predicated_udiv_no_profile'
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
@@ -113,12 +114,12 @@ define void @merged_regions_use_conservative_probability(ptr noalias %a, ptr noa
; CHECK-LABEL: 'merged_regions_use_conservative_probability'
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7:%[0-9]+]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6:%[0-9]+]]>, ir<%gep>.1
-; CHECK: Cost of 0 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6]]>, ir<%gep>.1
-; CHECK: Cost of 0 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
;
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index a135fa3a232f2..81ac481fbabb9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -17,10 +17,12 @@
#include "../lib/Transforms/Vectorize/VPlanTransforms.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/CycleInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/SourceMgr.h"
@@ -42,6 +44,8 @@ class VPlanTestIRBase : public testing::Test {
std::unique_ptr<ScalarEvolution> SE;
std::unique_ptr<TargetLibraryInfoImpl> TLII;
std::unique_ptr<TargetLibraryInfo> TLI;
+ std::unique_ptr<CycleInfo> CI;
+ std::unique_ptr<BranchProbabilityInfo> BPI;
MapVector<PHINode *, InductionDescriptor> Inductions;
@@ -65,6 +69,9 @@ class VPlanTestIRBase : public testing::Test {
LI.reset(new LoopInfo(*DT));
AC.reset(new AssumptionCache(F));
SE.reset(new ScalarEvolution(F, *TLI, *AC, *DT, *LI));
+ CI.reset(new CycleInfo());
+ CI->compute(F);
+ BPI.reset(new BranchProbabilityInfo(F, *CI, TLI.get(), DT.get()));
}
/// Build the VPlan for the loop starting from \p LoopHeader.
@@ -77,8 +84,9 @@ class VPlanTestIRBase : public testing::Test {
Loop *L = LI->getLoopFor(LoopHeader);
PredicatedScalarEvolution PSE(*SE, *L);
- auto Plan =
- VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), PSE);
+ auto Plan = VPlanTransforms::buildVPlan0(
+ L, *LI, IntegerType::get(*Ctx, 64), PSE, /*LVer=*/nullptr,
+ [this]() -> const BranchProbabilityInfo & { return *BPI; });
if (Style) {
Inductions.clear();
More information about the llvm-commits
mailing list