[llvm] [VPlan] Record execution frequencies on VPlan0. (PR #222337)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 10:48:22 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/222337
>From 01fc10bfba42ae5a2ed1134cb8a06ba2f225149d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 9 Sep 2026 13:52:19 +0100
Subject: [PATCH 1/4] [VPlan] Record execution frequencies on VPlan0.
Move recording of the execution frequencies from introduceMasksAndLinearize
into a separate recordExecutionFrequencies transform, run while VPlan0 still
mirrors the original loop's CFG.
This more cleanly separates it from the unrelated transform and allows
the frequencies to be used to compute the cost of scalar VPlans:
https://github.com/llvm/llvm-project/pull/196845
---
.../Transforms/Vectorize/LoopVectorize.cpp | 42 ++++---
.../Vectorize/VPlanConstruction.cpp | 12 ++
.../Transforms/Vectorize/VPlanPredicator.cpp | 18 ++-
.../Transforms/Vectorize/VPlanTransforms.h | 4 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 7 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 3 +-
.../VPlan/record-execution-frequencies.ll | 116 ++++++++++++++++++
.../VPlan/vplan-print-before-after-all.ll | 1 +
8 files changed, 170 insertions(+), 33 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 82b1faaaa9b23..8e4d4d3fc67d7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6292,26 +6292,22 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
static void printOptimizedVPlan(VPlan &) {}
#ifndef NDEBUG
-/// Cross-check vputils::computeExecutionFrequencies for the loop region of
-/// \p Plan against BlockFrequencyInfo for the blocks of \p OrigLoop.
+/// Cross-check the execution frequencies recorded in \p Plan against
+/// BlockFrequencyInfo for the blocks of \p OrigLoop.
/// FIXME: Temporary verification aid, to be removed.
static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
LoopInfo *LI,
LoopVectorizationCostModel &CM) {
- // Limited to inner loops with the latch as only exiting block and no extra
- // VPBBs without a matching IR BB (as introduced by tail folding).
- if (Plan.isOuterLoop() ||
- OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch() ||
- Plan.hasTailFolded())
+ // Limited to loops with the latch as only exiting block
+ if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
return true;
- // Visit the region's blocks in the same order as introduceMasksAndLinearize.
- // Both are reverse post-orders of the same CFG, so indices correspond.
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
- Plan.getVectorLoopRegion()->getEntryBasicBlock());
- auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
+ // Visit the loop body in the same order as recordExecutionFrequencies. Both
+ // are reverse post-orders of the same CFG, so indices correspond.
+ VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
+ SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
assert(Blocks.size() == OrigLoop->getNumBlocks() &&
- "loop region and original loop must have the same blocks");
+ "loop body and original loop must have the same blocks");
LoopBlocksRPO OrigRPO(OrigLoop);
OrigRPO.perform(LI);
@@ -6329,15 +6325,18 @@ static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
Edges += VPBB->getNumSuccessors();
uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
- DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
- Frequencies = vputils::computeExecutionFrequencies(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
- // Compare at BranchProbability's coarser resolution, which is as precise as
- // BFI's frequencies get.
- std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
+ // All recipes of a block share the same recorded frequency; empty blocks
+ // and blocks that always or never execute carry none.
+ if (VPBB->empty())
+ continue;
+ std::optional<VPExecutionFrequency> Freq =
+ cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
if (!Freq)
continue;
+ // Compare at BranchProbability's coarser resolution, which is as precise as
+ // BFI's frequencies get.
BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
// Clamp to the header's frequency, which BFI's rounding may exceed.
@@ -6387,6 +6386,11 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
LAI->getSymbolicStrides(), VPDT);
RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
+ if (IsInnerLoop) {
+ RUN_VPLAN_PASS(VPlanTransforms::recordExecutionFrequencies, *VPlan0);
+ assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, *CM) &&
+ "execution frequencies do not match the loop's block frequencies");
+ }
// Create recipes for header phis. For outer loops, reductions, recurrences
// and in-loop reductions are empty since legality doesn't detect them.
@@ -6442,8 +6446,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
if (CM->foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
- assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, *CM) &&
- "execution frequencies do not match the loop's block frequencies");
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 17a3b9790aadb..30133e51330ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -643,6 +643,18 @@ std::unique_ptr<VPlan> VPlanTransforms::buildVPlan0(
return VPlan0;
}
+void VPlanTransforms::recordExecutionFrequencies(VPlan &Plan) {
+ VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
+ SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
+ auto Frequencies = vputils::computeExecutionFrequencies(Blocks);
+ LLVMContext &Ctx = Plan.getContext();
+ for (VPBasicBlock *VPBB : Blocks) {
+ std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
+ for (VPRecipeBase &R : *VPBB)
+ cast<VPInstruction>(&R)->setExecutionFrequency(Freq, Ctx);
+ }
+}
+
/// Creates a VPWidenIntOrFpInductionRecipe or VPWidenPointerInductionRecipe
/// for \p Phi based on \p IndDesc.
static VPHeaderPHIRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index e056b2b613b33..ba927ffd21f33 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -406,8 +406,6 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Header);
// Non-outer regions with VPBBs only are supported at the moment.
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
- Frequencies = vputils::computeExecutionFrequencies(Blocks);
VPPredicator Predicator(Plan);
for (VPBasicBlock *VPBB : Blocks) {
@@ -418,19 +416,17 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Predicator.createBlockInMask(VPBB);
VPValue *BlockMask = Predicator.getBlockInMask(VPBB);
- if (!BlockMask)
- continue;
-
- // Mask all VPInstructions in the block and record the frequency with
- // which the masked recipes execute.
- std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
+ // Mask all VPInstructions in the block. The execution frequency recorded
+ // during VPlan0 construction only stays meaningful for recipes that remain
+ // predicated once the CFG is linearized below; drop it from the others.
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
continue;
- VPI->addMask(BlockMask);
- if (VPI->isMasked())
- VPI->setExecutionFrequency(Freq, Plan.getContext());
+ if (BlockMask)
+ VPI->addMask(BlockMask);
+ if (!VPI->isMasked())
+ VPI->clearExecutionFrequency();
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1c7b17942b795..6e5a2184270a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -160,6 +160,10 @@ struct VPlanTransforms {
PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr,
function_ref<const BranchProbabilityInfo &()> GetBPI = nullptr);
+ /// Add execution frequencies to each recipe in the loop body of \p Plan.
+ /// Frequencies are computed from the branch weights in \p Plan.
+ static void recordExecutionFrequencies(VPlan &Plan);
+
/// Replace VPPhi recipes in \p Plan's header with corresponding
/// VPHeaderPHIRecipe subclasses for inductions, reductions, and
/// fixed-order recurrences. This processes all header phis and creates
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 8327b30c7583e..7ed59db83884d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1242,7 +1242,12 @@ vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- std::optional<VPExecutionFrequency> &SuccFreq = Frequencies.at(Succ);
+ // Ignore edges leaving Blocks, i.e. a plain CFG's edges to the middle
+ // block or to an exit block.
+ auto It = Frequencies.find(Succ);
+ if (It == Frequencies.end())
+ continue;
+ std::optional<VPExecutionFrequency> &SuccFreq = It->second;
// An unknown edge or predecessor poisons the successor.
if (!Src || EdgeProb.isUnknown() || !SuccFreq) {
SuccFreq = std::nullopt;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 738a5bc8b8066..3b3c40043c05c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -235,7 +235,8 @@ BranchProbability getExecutionProbability(BlockFrequency Freq);
/// the frequency with which it executes relative to the first (header) block,
/// and whether that frequency was composed using any estimated branch weights.
/// The frequency of a block is the sum over its incoming edges, or std::nullopt
-/// if any edge on a path reaching it lacks branch weights.
+/// if any edge on a path reaching it lacks branch weights. Edges to blocks
+/// outside \p Blocks are ignored.
DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
new file mode 100644
index 0000000000000..e18ce8bef391a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
@@ -0,0 +1,116 @@
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 \
+; RUN: -vplan-print-after=recordExecutionFrequencies -disable-output %s 2>&1 \
+; RUN: | FileCheck --check-prefix=VPLAN0 %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 \
+; RUN: -vplan-print-after=introduceMasksAndLinearize -disable-output %s 2>&1 \
+; RUN: | FileCheck --check-prefix=MASKED %s
+
+; The recipes of %if.then all execute with the block's frequency of 1/4, the
+; ones of %loop and %latch always execute.
+define void @if_then(ptr noalias %a, ptr noalias %idx) {
+; VPLAN0-LABEL: VPlan for loop in 'if_then'
+; VPLAN0: loop:
+; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
+; VPLAN0-NEXT: Successor(s): if.then, latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: if.then:
+; VPLAN0-NEXT: EMIT ir<%add> = add ir<%i>, ir<10> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT store ir<%add>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: latch:
+; VPLAN0-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN0-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+; After predication, %gep.a is executed unconditionally.
+;
+; MASKED-LABEL: VPlan for loop in 'if_then'
+; MASKED: if.then:
+; MASKED-NEXT: EMIT ir<%add> = add ir<%i>, ir<10>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
+; MASKED-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>{{$}}
+; MASKED-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
+; MASKED-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c = icmp sgt i32 %i, 0
+ br i1 %c, label %if.then, label %latch, !prof !0
+
+if.then:
+ %add = add i32 %i, 10
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %add, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+; Both arms of the diamond are conditionally executed, with the frequencies of
+; 1/4 and 3/4 recorded on their respective recipes.
+define void @if_else(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; VPLAN0-LABEL: VPlan for loop in 'if_else'
+; VPLAN0: loop:
+; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
+; VPLAN0-NEXT: Successor(s): then, else
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: else:
+; VPLAN0-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.b> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: then:
+; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c = icmp sgt i32 %i, 0
+ br i1 %c, label %then, label %else, !prof !0
+
+then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %i, ptr %gep.a, align 4
+ br label %latch
+
+else:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %i, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 3}
+!1 = !{!"branch_weights", i32 1, i32 999}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index b5ee488ae5bd5..fe71cee7c3492 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -14,6 +14,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::combineRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::removeDeadRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::recordExecutionFrequencies
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceSymbolicStrides at 2
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::finalizeSCEVPredicates
>From 7e871646cc709e22465172f96a982ad5bdcf6294 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 10 Sep 2026 18:00:51 +0100
Subject: [PATCH 2/4] !fixup adjust comments, checks
---
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp | 7 ++++---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 13 +++++++------
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index d99be7f53a413..b4f0f9ee78e18 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -417,15 +417,16 @@ void VPPredicator::run() {
createBlockInMask(VPBB);
VPValue *BlockMask = getBlockInMask(VPBB);
- // Mask all VPInstructions in the block. The execution frequency recorded
- // during VPlan0 construction only stays meaningful for recipes that remain
- // predicated once the CFG is linearized below; drop it from the others.
+ // Mask all VPInstructions in the block.
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
continue;
if (BlockMask)
VPI->addMask(BlockMask);
+
+ // Drop the execution frequency of unmasked VPInstructions, as they
+ // always execute.
if (!VPI->isMasked())
VPI->clearExecutionFrequency();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 7ed59db83884d..43aff68f2992a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1225,9 +1225,9 @@ static BlockFrequency scaleKeepingNonZero(BlockFrequency Freq,
DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
assert(!Blocks.empty() && "expected at least the header block");
- // Push each block's frequency along its outgoing edges. Blocks is a DAG in
- // reverse post-order (the loop region's backedge is implicit), so a block's
- // frequency is final by the time it is visited.
+ // Push each block's frequency along its outgoing edges. Blocks is in reverse
+ // post-order and forms a DAG with the backedge from the latch (the last
+ // block) ignored, so a block's frequency is final by the time it is visited.
DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
Frequencies;
Frequencies.reserve(Blocks.size());
@@ -1242,10 +1242,11 @@ vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- // Ignore edges leaving Blocks, i.e. a plain CFG's edges to the middle
- // block or to an exit block.
+ // Ignore the backedge to the header (already treated as always executing)
+ // and edges leaving Blocks, i.e. a plain CFG's edges to the middle block
+ // or to an exit block.
auto It = Frequencies.find(Succ);
- if (It == Frequencies.end())
+ if (Succ == Blocks.front() || It == Frequencies.end())
continue;
std::optional<VPExecutionFrequency> &SuccFreq = It->second;
// An unknown edge or predecessor poisons the successor.
>From b1ef3a749d66bec3472b23379ef03b0b3efee901 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Sep 2026 16:22:09 +0100
Subject: [PATCH 3/4] !fixup update to use make_isa_range
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 -
.../Vectorize/VPlanConstruction.cpp | 4 +-
.../replicate-region-guard-unknown-prof.ll | 56 +++++++++----------
3 files changed, 30 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0f10ba1124a43..680a5b113c80f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6344,8 +6344,6 @@ static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
if (!Freq)
continue;
- // Compare at BranchProbability's coarser resolution, which is as precise as
- // BFI's frequencies get.
BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
// Clamp to the header's frequency, which BFI's rounding may exceed.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index e013868368ff0..036c3fc496e9e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -636,8 +636,8 @@ void VPlanTransforms::recordExecutionFrequencies(VPlan &Plan) {
LLVMContext &Ctx = Plan.getContext();
for (VPBasicBlock *VPBB : Blocks) {
std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
- for (VPRecipeBase &R : *VPBB)
- cast<VPInstruction>(&R)->setExecutionFrequency(Freq, Ctx);
+ for (VPInstruction &VPI : make_isa_range<VPInstruction>(*VPBB))
+ VPI.setExecutionFrequency(Freq, Ctx);
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-guard-unknown-prof.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-guard-unknown-prof.ll
index 9fedfd49bc2b7..571a2f5a0f113 100644
--- a/llvm/test/Transforms/LoopVectorize/replicate-region-guard-unknown-prof.ll
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-guard-unknown-prof.ll
@@ -22,19 +22,19 @@ define void @tail_folded_predicated_store(ptr noalias %a, ptr noalias %b, i64 %n
; CHECK: [[PRED_LOAD_IF5]]:
; CHECK: [[PRED_LOAD_CONTINUE6]]:
; CHECK: [[TMP26:%.*]] = select <4 x i1> [[TMP1:%.*]], <4 x i1> [[TMP25:%.*]], <4 x i1> zeroinitializer
-; CHECK: br i1 [[TMP27:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: br i1 [[TMP27:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK: br i1 [[TMP30:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK: br i1 [[TMP30:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF7]]:
; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK: br i1 [[TMP33:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK: br i1 [[TMP33:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF9]]:
; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK: br i1 [[TMP36:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; CHECK: br i1 [[TMP36:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF11]]:
; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK: br i1 [[TMP39:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: br i1 [[TMP39:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK: [[EXIT:.*:]]
;
@@ -67,27 +67,27 @@ define void @predicated_store(ptr noalias %a, ptr noalias %b, i64 %n) !prof !0 {
; CHECK-LABEL: define void @predicated_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
; CHECK: [[ENTRY:.*:]]
-; CHECK: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF6:![0-9]+]]
+; CHECK: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF7:![0-9]+]]
; CHECK: [[VECTOR_PH]]:
; CHECK: [[VECTOR_BODY:.*]]:
-; CHECK: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7:![0-9]+]]
+; CHECK: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF7]]
+; CHECK: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF7]]
+; CHECK: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF3]]:
; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF7]]
+; CHECK: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF5]]:
; CHECK: [[PRED_STORE_CONTINUE6]]:
-; CHECK: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
+; CHECK: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF1]]
; CHECK: [[SCALAR_PH]]:
; CHECK: [[LOOP_HEADER:.*]]:
-; CHECK: br i1 [[C:%.*]], label %[[THEN:.*]], label %[[LOOP_LATCH:.*]], !prof [[PROF7]]
+; CHECK: br i1 [[C:%.*]], label %[[THEN:.*]], label %[[LOOP_LATCH:.*]], !prof [[PROF1]]
; CHECK: [[THEN]]:
; CHECK: [[LOOP_LATCH]]:
; CHECK: br i1 [[EC:%.*]], label %[[EXIT]], label %[[LOOP_HEADER]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -136,19 +136,19 @@ define void @tail_folded_predicated_store_not_profiled(ptr noalias %a, ptr noali
; CHECK: [[PRED_LOAD_IF5]]:
; CHECK: [[PRED_LOAD_CONTINUE6]]:
; CHECK: [[TMP26:%.*]] = select <4 x i1> [[TMP1:%.*]], <4 x i1> [[TMP25:%.*]], <4 x i1> zeroinitializer
-; CHECK: br i1 [[TMP27:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: br i1 [[TMP27:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK: br i1 [[TMP30:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK: br i1 [[TMP30:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF7]]:
; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK: br i1 [[TMP33:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK: br i1 [[TMP33:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF9]]:
; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK: br i1 [[TMP36:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; CHECK: br i1 [[TMP36:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]], !prof [[PROF1]]
; CHECK: [[PRED_STORE_IF11]]:
; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK: br i1 [[TMP39:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: br i1 [[TMP39:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK: [[EXIT:.*:]]
;
@@ -181,16 +181,16 @@ exit:
!2 = !{!"branch_weights", i32 1, i32 99}
;.
; CHECK: [[PROF0]] = !{!"function_entry_count", i64 1000}
-; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 24}
-; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
-; CHECK: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 25}
-; CHECK: [[PROF6]] = !{!"branch_weights", i32 1, i32 127}
-; CHECK: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META4]], [[META5]]}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 1, i32 24}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 25}
+; CHECK: [[PROF7]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META5]], [[META6]]}
; CHECK: [[PROF9]] = !{!"branch_weights", i32 1, i32 0}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]], [[META3]], [[META11:![0-9]+]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]], [[META4]], [[META11:![0-9]+]]}
; CHECK: [[META11]] = !{!"llvm.loop.estimated_trip_count", i32 0}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]], [[META4]], [[META5]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META5]], [[META6]]}
;.
>From 4597442709c02ab74cecd7c90234e8b730ecdf79 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Sep 2026 18:32:55 +0100
Subject: [PATCH 4/4] !fixup auto-generate checks, early-continue
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 11 +-
.../VPlan/record-execution-frequencies.ll | 175 ++++++++++++++----
2 files changed, 148 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 40473cf2d78c0..15e1feaf5908c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1243,11 +1243,14 @@ vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- // Ignore the backedge to the header (already treated as always executing)
- // and edges leaving Blocks, i.e. a plain CFG's edges to the middle block
- // or to an exit block.
+ // Ignore the backedge to the header (already treated as always
+ // executing).
+ if (Succ == Blocks.front())
+ continue;
+ // Ignore edges leaving Blocks, i.e. a plain CFG's edges to the middle
+ // block or to an exit block.
auto It = Frequencies.find(Succ);
- if (Succ == Blocks.front() || It == Frequencies.end())
+ if (|| It == Frequencies.end())
continue;
std::optional<VPExecutionFrequency> &SuccFreq = It->second;
// An unknown edge or predecessor poisons the successor.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
index e18ce8bef391a..c3f3e873d1567 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/record-execution-frequencies.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 \
; RUN: -vplan-print-after=recordExecutionFrequencies -disable-output %s 2>&1 \
; RUN: | FileCheck --check-prefix=VPLAN0 %s
@@ -9,32 +10,76 @@
; ones of %loop and %latch always execute.
define void @if_then(ptr noalias %a, ptr noalias %idx) {
; VPLAN0-LABEL: VPlan for loop in 'if_then'
-; VPLAN0: loop:
-; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
-; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
-; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
-; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
-; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
-; VPLAN0-NEXT: Successor(s): if.then, latch
+; VPLAN0: VPlan ' for UF>=1' {
+; VPLAN0-NEXT: Live-in ir<1024> = original trip-count
; VPLAN0-EMPTY:
-; VPLAN0-NEXT: if.then:
-; VPLAN0-NEXT: EMIT ir<%add> = add ir<%i>, ir<10> (!vplan.execution.frequency 2305843009213693952 (25%))
-; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
-; VPLAN0-NEXT: EMIT store ir<%add>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
-; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-NEXT: ir-bb<entry>:
+; VPLAN0-NEXT: Successor(s): scalar.ph, vector.ph
; VPLAN0-EMPTY:
-; VPLAN0-NEXT: latch:
-; VPLAN0-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
-; VPLAN0-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
-;
-; After predication, %gep.a is executed unconditionally.
+; VPLAN0-NEXT: vector.ph:
+; VPLAN0-NEXT: Successor(s): loop
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: loop:
+; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
+; VPLAN0-NEXT: Successor(s): if.then, latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: if.then:
+; VPLAN0-NEXT: EMIT ir<%add> = add ir<%i>, ir<10> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT store ir<%add>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: latch:
+; VPLAN0-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN0-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%ec> (!prof {1, 999})
+; VPLAN0-NEXT: Successor(s): middle.block, loop
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: middle.block:
;
; MASKED-LABEL: VPlan for loop in 'if_then'
-; MASKED: if.then:
+; MASKED: VPlan ' for UF>=1' {
+; MASKED-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; MASKED-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; MASKED-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; MASKED-NEXT: Live-in ir<1024> = original trip-count
+; MASKED-EMPTY:
+; MASKED-NEXT: ir-bb<entry>:
+; MASKED-NEXT: Successor(s): scalar.ph, vector.ph
+; MASKED-EMPTY:
+; MASKED-NEXT: vector.ph:
+; MASKED-NEXT: Successor(s): vector loop
+; MASKED-EMPTY:
+; MASKED-NEXT: <x1> vector loop: {
+; MASKED-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; MASKED-EMPTY:
+; MASKED-NEXT: vector.body:
+; MASKED-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; MASKED-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; MASKED-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; MASKED-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; MASKED-NEXT: Successor(s): if.then
+; MASKED-EMPTY:
+; MASKED-NEXT: if.then:
; MASKED-NEXT: EMIT ir<%add> = add ir<%i>, ir<10>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
-; MASKED-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>{{$}}
+; MASKED-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; MASKED-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
; MASKED-NEXT: Successor(s): latch
+; MASKED-EMPTY:
+; MASKED-NEXT: latch:
+; MASKED-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; MASKED-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; MASKED-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; MASKED-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; MASKED-NEXT: No successors
+; MASKED-NEXT: }
+; MASKED-NEXT: Successor(s): middle.block
+; MASKED-EMPTY:
+; MASKED-NEXT: middle.block:
;
entry:
br label %loop
@@ -65,23 +110,85 @@ exit:
; 1/4 and 3/4 recorded on their respective recipes.
define void @if_else(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN0-LABEL: VPlan for loop in 'if_else'
-; VPLAN0: loop:
-; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
-; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
-; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
-; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
-; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
-; VPLAN0-NEXT: Successor(s): then, else
+; VPLAN0: VPlan ' for UF>=1' {
+; VPLAN0-NEXT: Live-in ir<1024> = original trip-count
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: ir-bb<entry>:
+; VPLAN0-NEXT: Successor(s): scalar.ph, vector.ph
; VPLAN0-EMPTY:
-; VPLAN0-NEXT: else:
-; VPLAN0-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv> (!vplan.execution.frequency 6917529027641081856 (75%))
-; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.b> (!vplan.execution.frequency 6917529027641081856 (75%))
-; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-NEXT: vector.ph:
+; VPLAN0-NEXT: Successor(s): loop
; VPLAN0-EMPTY:
-; VPLAN0-NEXT: then:
-; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
-; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
-; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-NEXT: loop:
+; VPLAN0-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; VPLAN0-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN0-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN0-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%c> (!prof {1, 3})
+; VPLAN0-NEXT: Successor(s): then, else
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: else:
+; VPLAN0-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.b> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: then:
+; VPLAN0-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: EMIT store ir<%i>, ir<%gep.a> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN0-NEXT: Successor(s): latch
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: latch:
+; VPLAN0-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN0-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; VPLAN0-NEXT: EMIT branch-on-cond ir<%ec> (!prof {1, 999})
+; VPLAN0-NEXT: Successor(s): middle.block, loop
+; VPLAN0-EMPTY:
+; VPLAN0-NEXT: middle.block:
+;
+; MASKED-LABEL: VPlan for loop in 'if_else'
+; MASKED: VPlan ' for UF>=1' {
+; MASKED-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; MASKED-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; MASKED-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; MASKED-NEXT: Live-in ir<1024> = original trip-count
+; MASKED-EMPTY:
+; MASKED-NEXT: ir-bb<entry>:
+; MASKED-NEXT: Successor(s): scalar.ph, vector.ph
+; MASKED-EMPTY:
+; MASKED-NEXT: vector.ph:
+; MASKED-NEXT: Successor(s): vector loop
+; MASKED-EMPTY:
+; MASKED-NEXT: <x1> vector loop: {
+; MASKED-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; MASKED-EMPTY:
+; MASKED-NEXT: vector.body:
+; MASKED-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; MASKED-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; MASKED-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; MASKED-NEXT: EMIT ir<%c> = icmp sgt ir<%i>, ir<0>
+; MASKED-NEXT: Successor(s): else
+; MASKED-EMPTY:
+; MASKED-NEXT: else:
+; MASKED-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c>
+; MASKED-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; MASKED-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[VP4]]> (!vplan.execution.frequency 6917529027641081856 (75%))
+; MASKED-NEXT: Successor(s): then
+; MASKED-EMPTY:
+; MASKED-NEXT: then:
+; MASKED-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; MASKED-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c> (!vplan.execution.frequency 2305843009213693952 (25%))
+; MASKED-NEXT: Successor(s): latch
+; MASKED-EMPTY:
+; MASKED-NEXT: latch:
+; MASKED-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; MASKED-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; MASKED-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; MASKED-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; MASKED-NEXT: No successors
+; MASKED-NEXT: }
+; MASKED-NEXT: Successor(s): middle.block
+; MASKED-EMPTY:
+; MASKED-NEXT: middle.block:
;
entry:
br label %loop
More information about the llvm-commits
mailing list