[llvm] [VPlan] Record execution frequencies on VPlan0. (PR #222337)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 07:08:30 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/222337
Move recording of the execution frequencies from introduceMasksAndLinearize into a separate recordExecutionFrequencies transform, run while VPlan0 still mirrors the original loop's CFG.
This more cleanly separates it from the unrelated transform and allows the frequencies to be used to compute the cost of scalar VPlans: https://github.com/llvm/llvm-project/pull/196845
>From e221f2541a16bfe35e6bf325508eb9b1ce4283df Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 9 Sep 2026 13:52:19 +0100
Subject: [PATCH] [VPlan] Record execution frequencies on VPlan0.
Move recording of the execution frequencies from introduceMasksAndLinearize
into a separate recordExecutionFrequencies transform, run while VPlan0 still
mirrors the original loop's CFG.
This more cleanly separates it from the unrelated transform and allows
the frequencies to be used to compute the cost of scalar VPlans:
https://github.com/llvm/llvm-project/pull/196845
---
.../Transforms/Vectorize/LoopVectorize.cpp | 43 ++++---
.../Vectorize/VPlanConstruction.cpp | 12 ++
.../Transforms/Vectorize/VPlanPredicator.cpp | 18 ++-
.../Transforms/Vectorize/VPlanTransforms.h | 4 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 7 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 3 +-
.../VPlan/record-execution-frequencies.ll | 116 ++++++++++++++++++
.../VPlan/vplan-print-before-after-all.ll | 1 +
8 files changed, 171 insertions(+), 33 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 82b1faaaa9b23..0e75ef0f1b41f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6292,26 +6292,23 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
static void printOptimizedVPlan(VPlan &) {}
#ifndef NDEBUG
-/// Cross-check vputils::computeExecutionFrequencies for the loop region of
-/// \p Plan against BlockFrequencyInfo for the blocks of \p OrigLoop.
+/// Cross-check the execution frequencies recorded in \p Plan against
+/// BlockFrequencyInfo for the blocks of \p OrigLoop.
/// FIXME: Temporary verification aid, to be removed.
static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
LoopInfo *LI,
LoopVectorizationCostModel &CM) {
- // Limited to inner loops with the latch as only exiting block and no extra
- // VPBBs without a matching IR BB (as introduced by tail folding).
- if (Plan.isOuterLoop() ||
- OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch() ||
- Plan.hasTailFolded())
+ assert(!Plan.isOuterLoop() && "outer loops not supported");
+ // Limited to loops with the latch as only exiting block
+ if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
return true;
- // Visit the region's blocks in the same order as introduceMasksAndLinearize.
- // Both are reverse post-orders of the same CFG, so indices correspond.
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
- Plan.getVectorLoopRegion()->getEntryBasicBlock());
- auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
+ // Visit the loop body in the same order as recordExecutionFrequencies. Both
+ // are reverse post-orders of the same CFG, so indices correspond.
+ VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
+ SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
assert(Blocks.size() == OrigLoop->getNumBlocks() &&
- "loop region and original loop must have the same blocks");
+ "loop body and original loop must have the same blocks");
LoopBlocksRPO OrigRPO(OrigLoop);
OrigRPO.perform(LI);
@@ -6329,15 +6326,18 @@ static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
Edges += VPBB->getNumSuccessors();
uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
- DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
- Frequencies = vputils::computeExecutionFrequencies(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
- // Compare at BranchProbability's coarser resolution, which is as precise as
- // BFI's frequencies get.
- std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
+ // All recipes of a block share the same recorded frequency; empty blocks
+ // and blocks that always or never execute carry none.
+ if (VPBB->empty())
+ continue;
+ std::optional<VPExecutionFrequency> Freq =
+ cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
if (!Freq)
continue;
+ // Compare at BranchProbability's coarser resolution, which is as precise as
+ // BFI's frequencies get.
BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
// Clamp to the header's frequency, which BFI's rounding may exceed.
@@ -6387,6 +6387,11 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
LAI->getSymbolicStrides(), VPDT);
RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
+ if (IsInnerLoop) {
+ RUN_VPLAN_PASS(VPlanTransforms::recordExecutionFrequencies, *VPlan0);
+ assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, *CM) &&
+ "execution frequencies do not match the loop's block frequencies");
+ }
// Create recipes for header phis. For outer loops, reductions, recurrences
// and in-loop reductions are empty since legality doesn't detect them.
@@ -6442,8 +6447,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
if (CM->foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
- assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, *CM) &&
- "execution frequencies do not match the loop's block frequencies");
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 17a3b9790aadb..30133e51330ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -643,6 +643,18 @@ std::unique_ptr<VPlan> VPlanTransforms::buildVPlan0(
return VPlan0;
}
+void VPlanTransforms::recordExecutionFrequencies(VPlan &Plan) {
+ VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
+ SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
+ auto Frequencies = vputils::computeExecutionFrequencies(Blocks);
+ LLVMContext &Ctx = Plan.getContext();
+ for (VPBasicBlock *VPBB : Blocks) {
+ std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
+ for (VPRecipeBase &R : *VPBB)
+ cast<VPInstruction>(&R)->setExecutionFrequency(Freq, Ctx);
+ }
+}
+
/// Creates a VPWidenIntOrFpInductionRecipe or VPWidenPointerInductionRecipe
/// for \p Phi based on \p IndDesc.
static VPHeaderPHIRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index e056b2b613b33..ba927ffd21f33 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -406,8 +406,6 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Header);
// Non-outer regions with VPBBs only are supported at the moment.
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
- Frequencies = vputils::computeExecutionFrequencies(Blocks);
VPPredicator Predicator(Plan);
for (VPBasicBlock *VPBB : Blocks) {
@@ -418,19 +416,17 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Predicator.createBlockInMask(VPBB);
VPValue *BlockMask = Predicator.getBlockInMask(VPBB);
- if (!BlockMask)
- continue;
-
- // Mask all VPInstructions in the block and record the frequency with
- // which the masked recipes execute.
- std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
+ // Mask all VPInstructions in the block. The execution frequency recorded
+ // during VPlan0 construction only stays meaningful for recipes that remain
+ // predicated once the CFG is linearized below; drop it from the others.
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
continue;
- VPI->addMask(BlockMask);
- if (VPI->isMasked())
- VPI->setExecutionFrequency(Freq, Plan.getContext());
+ if (BlockMask)
+ VPI->addMask(BlockMask);
+ if (!VPI->isMasked())
+ VPI->clearExecutionFrequency();
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1c7b17942b795..6e5a2184270a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -160,6 +160,10 @@ struct VPlanTransforms {
PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr,
function_ref<const BranchProbabilityInfo &()> GetBPI = nullptr);
+ /// Add execution frequencies to each recipe in the loop body of \p Plan.
+ /// Frequencies are computed from the branch weights in \p Plan.
+ static void recordExecutionFrequencies(VPlan &Plan);
+
/// Replace VPPhi recipes in \p Plan's header with corresponding
/// VPHeaderPHIRecipe subclasses for inductions, reductions, and
/// fixed-order recurrences. This processes all header phis and creates
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 8327b30c7583e..7ed59db83884d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1242,7 +1242,12 @@ vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- std::optional<VPExecutionFrequency> &SuccFreq = Frequencies.at(Succ);
+ // Ignore edges leaving Blocks, i.e. a plain CFG's edges to the middle
+ // block or to an exit block.
+ auto It = Frequencies.find(Succ);
+ if (It == Frequencies.end())
+ continue;
+ std::optional<VPExecutionFrequency> &SuccFreq = It->second;
// An unknown edge or predecessor poisons the successor.
if (!Src || EdgeProb.isUnknown() || !SuccFreq) {
SuccFreq = std::nullopt;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 738a5bc8b8066..3b3c40043c05c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -235,7 +235,8 @@ BranchProbability getExecutionProbability(BlockFrequency Freq);
/// the frequency with which it executes relative to the first (header) block,
/// and whether that frequency was composed using any estimated branch weights.
/// The frequency of a block is the sum over its incoming edges, or std::nullopt
-/// if any edge on a path reaching it lacks branch weights.
+/// if any edge on a path reaching it lacks branch weights. Edges to blocks
+/// outside \p Blocks are ignored.
DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
new file mode 100644
index 0000000000000..e18ce8bef391a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
@@ -0,0 +1,116 @@
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 \
+; RUN: -vplan-print-after=recordExecutionFrequencies -disable-output %s 2>&1 \
+; RUN: | FileCheck --check-prefix=VPLAN0 %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 \
+; RUN: -vplan-print-after=introduceMasksAndLinearize -disable-output %s 2>&1 \
+; RUN: | FileCheck --check-prefix=MASKED %s
+
+; The recipes of %if.then all execute with the block's frequency of 1/4, the
+; ones of %loop and %latch always execute.
+define void @if_then(ptr noalias %a, ptr noalias %idx) {
+; VPLAN0-LABEL: VPlan for loop in 'if_then'
+; VPLAN0: loop:
+; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
+; VPLAN0-NEXT: Successor(s): if.then, latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: if.then:
+; VPLAN0-NEXT: EMIT ir<%add> = add ir<%i>, ir<10> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT store ir<%add>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: latch:
+; VPLAN0-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN0-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+; After predication, %gep.a is executed unconditionally.
+;
+; MASKED-LABEL: VPlan for loop in 'if_then'
+; MASKED: if.then:
+; MASKED-NEXT: EMIT ir<%add> = add ir<%i>, ir<10>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
+; MASKED-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>{{$}}
+; MASKED-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
+; MASKED-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c = icmp sgt i32 %i, 0
+ br i1 %c, label %if.then, label %latch, !prof !0
+
+if.then:
+ %add = add i32 %i, 10
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %add, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+; Both arms of the diamond are conditionally executed, with the frequencies of
+; 1/4 and 3/4 recorded on their respective recipes.
+define void @if_else(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; VPLAN0-LABEL: VPlan for loop in 'if_else'
+; VPLAN0: loop:
+; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
+; VPLAN0-NEXT: Successor(s): then, else
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: else:
+; VPLAN0-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.b> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: then:
+; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c = icmp sgt i32 %i, 0
+ br i1 %c, label %then, label %else, !prof !0
+
+then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %i, ptr %gep.a, align 4
+ br label %latch
+
+else:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %i, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 3}
+!1 = !{!"branch_weights", i32 1, i32 999}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index b5ee488ae5bd5..fe71cee7c3492 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -14,6 +14,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::combineRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeDeadRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::recordExecutionFrequencies
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::finalizeSCEVPredicates
More information about the llvm-commits
mailing list