[llvm] [VPlan] Preserve branch weights from VPlan0 through to codegen. (PR #213143)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 06:24:34 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/213143
>From 5071cca7f330020ac7c5cf7bfdda6600226a54c1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 29 Jul 2026 09:26:50 +0100
Subject: [PATCH 1/7] [VPlan] Preserve branch weights from VPlan0 through to
codegen.
This patch adds support for carrying through branch weights from the
original scalar loop through VPlan.
The flow is as follows:
1. On the scalar VPlan, we import the branch weights from the existing
IR branches and add them as !prof metadata to VPInstruction
terminators
2. The predicator dissolves the control flow and each block will get
executed unconditionally. What remains are predicated recipes, with
their lanes executing with probability relative to the header.
Compute the executing probabilities as sum of probabilities of all
incoming edges.
3. Each masked recipe gets the execution probability attached as !prof
metadata (similar to LLVM IR instructions like selects)
4. After unmasking recipes, a simple pass drops all !prof metadata from
recipe that are no longer predicated (currently they only remove on
predicated VPReplicateRecipes which get expanded to branch-based CFG
later, but in the future we may also retain it for other recipes that
can preserve it, like selects)
5. When creating replicate regions, a predicated recipe is replaced by
explicit control fold. The profile metadata from the recipe is
transferred to the newly introduced conditional branch.
6. During ::execute, !prof metadata is add to the generated instructions
using the existing metadata logic.
One thing to note is that step 2 duplicates some logic from IR-based
branch probability info, but doing it natively VPlan allows this to work
with CFG introduced by VPlan itself.
This is an alternative to https://github.com/llvm/llvm-project/pull/209309,
which carried to branch weights on VPBasicBlocks. This worked somewhat
by accident for replicate recipes: the predicator updates the CFG so all
blocks execute unconditionally, but preserved the execution probability.
This is incorrect at the VPlan level: the blocks execute
unconditionally after predication. It worked for the current use cases
because replicate regions are created before blocks are merged into
their predecessors, but is fragile and inaccurate. Carrying it
explicitly across the conditionally executed entities seems more
accurate.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 32 +-
.../Transforms/Vectorize/VPlanPredicator.cpp | 44 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 31 ++
.../Transforms/Vectorize/VPlanTransforms.h | 4 +
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 5 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 74 +++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 13 +
.../VPlan/vplan-print-before-after-all.ll | 1 +
.../VPlan/vplan-printing-branch-weights.ll | 10 +-
.../replicate-region-branch-weights.ll | 482 +++++++++---------
12 files changed, 446 insertions(+), 259 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 28237c7d037d4..93cfaaacc9c27 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6834,6 +6834,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
+ RUN_VPLAN_PASS(VPlanTransforms::dropBranchWeightsFromUnguardedRecipes, *Plan);
RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
Range);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..f326d088b1c49 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1184,7 +1184,12 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
/// Adds metatadata that can be preserved from the original instruction
/// \p I.
- VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
+ VPIRMetadata(Instruction &I) {
+ getMetadataToPropagate(&I, Metadata);
+ if (I.isTerminator())
+ if (MDNode *BW = I.getMetadata(LLVMContext::MD_prof))
+ Metadata.emplace_back(LLVMContext::MD_prof, BW);
+ }
/// Copy constructor for cloning.
VPIRMetadata(const VPIRMetadata &Other) = default;
@@ -1211,6 +1216,11 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
/// nodes that are common to both.
void intersect(const VPIRMetadata &MD);
+ /// Remove metadata of kind \p Kind, if present.
+ void eraseMetadata(unsigned Kind) {
+ erase_if(Metadata, [Kind](const auto &P) { return P.first == Kind; });
+ }
+
/// Get metadata of kind \p Kind. Returns nullptr if not found.
MDNode *getMetadata(unsigned Kind) const {
auto It =
@@ -3493,13 +3503,16 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
};
/// A recipe for generating conditional branches on the bits of a mask.
-class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase {
+class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase,
+ public VPIRMetadata {
public:
- VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL)
- : VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, {BlockInMask}, DL) {}
+ VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL,
+ const VPIRMetadata &Metadata = {})
+ : VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, {BlockInMask}, DL),
+ VPIRMetadata(Metadata) {}
VPBranchOnMaskRecipe *clone() override {
- return new VPBranchOnMaskRecipe(getOperand(0), getDebugLoc());
+ return new VPBranchOnMaskRecipe(getOperand(0), getDebugLoc(), *this);
}
VP_CLASSOF_IMPL(VPRecipeBase::VPBranchOnMaskSC)
@@ -4359,10 +4372,11 @@ struct CastInfo<VPWidenMemoryRecipe, const VPRecipeBase *>
/// Support casting from VPRecipeBase -> VPIRMetadata.
template <>
struct CastInfo<VPIRMetadata, VPRecipeBase *>
- : vpdetail::CastInfoMixinImpl<
- VPIRMetadata, VPInstruction, VPWidenRecipe, VPWidenCastRecipe,
- VPWidenIntrinsicRecipe, VPWidenCallRecipe, VPReplicateRecipe,
- VPInterleaveBase, VPWidenMemoryRecipe, VPHistogramRecipe> {};
+ : vpdetail::CastInfoMixinImpl<VPIRMetadata, VPInstruction, VPWidenRecipe,
+ VPWidenCastRecipe, VPWidenIntrinsicRecipe,
+ VPWidenCallRecipe, VPReplicateRecipe,
+ VPInterleaveBase, VPWidenMemoryRecipe,
+ VPHistogramRecipe, VPBranchOnMaskRecipe> {};
template <>
struct CastInfo<VPIRMetadata, const VPRecipeBase *>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 3f35f14e876f2..e69447bc87d66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -19,6 +19,8 @@
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/IR/MDBuilder.h"
+#include <numeric>
using namespace llvm;
using namespace VPlanPatternMatch;
@@ -112,6 +114,23 @@ class VPPredicator {
};
} // namespace
+/// Returns the branch weights describing how often a block executing with
+/// probability \p Prob is entered, or nullptr if \p Prob carries no useful
+/// information, i.e. the block is never or always executed, or its probability
+/// is unknown.
+static MDNode *getWeightsForProbability(BranchProbability Prob,
+ LLVMContext &Ctx) {
+ if (Prob.isUnknown() || Prob.isZero() || Prob.isOne())
+ return nullptr;
+
+ // Use the numerators of Prob and its complement as weights and reduce them
+ // via gcd to keep them small.
+ uint32_t Taken = Prob.getNumerator();
+ uint32_t NotTaken = Prob.getCompl().getNumerator();
+ uint32_t GCD = std::gcd(Taken, NotTaken);
+ return MDBuilder(Ctx).createBranchWeights(Taken / GCD, NotTaken / GCD);
+}
+
VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src,
const VPBasicBlock *Dst) {
assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge");
@@ -404,10 +423,13 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Header);
+ // Non-outer regions with VPBBs only are supported at the moment.
+ auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
+ DenseMap<const VPBasicBlock *, BranchProbability> Probabilities =
+ vputils::computeBlockProbabilities(Blocks);
+
VPPredicator Predicator(Plan);
- for (VPBlockBase *VPB : RPOT) {
- // Non-outer regions with VPBBs only are supported at the moment.
- auto *VPBB = cast<VPBasicBlock>(VPB);
+ for (VPBasicBlock *VPBB : Blocks) {
// Introduce the mask for VPBB, which may introduce needed edge masks, and
// convert all phi recipes of VPBB to blend recipes unless VPBB is the
// header.
@@ -418,20 +440,26 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
if (!BlockMask)
continue;
- // Mask all VPInstructions in the block.
+ // Mask all VPInstructions in the block and propagate execution
+ // probabilities to them.
+ MDNode *Weights =
+ getWeightsForProbability(Probabilities.lookup(VPBB), Plan.getContext());
for (VPRecipeBase &R : *VPBB) {
- if (auto *VPI = dyn_cast<VPInstruction>(&R))
+ if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
VPI->addMask(BlockMask);
+ if (Weights && VPI->isMasked())
+ VPI->setMetadata(LLVMContext::MD_prof, Weights);
+ }
}
}
- for (VPBlockBase *VPBB : reverse(RPOT))
+ for (VPBasicBlock *VPBB : reverse(Blocks))
if (VPBB != Header)
- Predicator.convertPhisToBlends(cast<VPBasicBlock>(VPBB));
+ Predicator.convertPhisToBlends(VPBB);
// Linearize the blocks of the loop into one serial chain.
VPBlockBase *PrevVPBB = nullptr;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ for (VPBasicBlock *VPBB : Blocks) {
auto Successors = to_vector(VPBB->getSuccessors());
if (Successors.size() > 1)
VPBB->getTerminator()->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ca63d1498316b..eae5d98a4f3cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -29,6 +29,7 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
@@ -2139,6 +2140,13 @@ void VPIRMetadata::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
assert(Kind < MDNames.size() && !MDNames[Kind].empty() &&
"Unexpected unnamed metadata kind");
O << "!" << MDNames[Kind] << " ";
+ SmallVector<uint32_t> Weights;
+ if (Kind == LLVMContext::MD_prof && extractBranchWeights(Node, Weights)) {
+ O << "{";
+ interleaveComma(Weights, O);
+ O << "}";
+ return;
+ }
Node->printAsOperand(O, M);
});
O << ")";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 412f120b0de1a..b255c39badfd1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -477,6 +477,15 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
if (!Then1 || !Then2)
continue;
+ // The merged region is entered whenever either of the original regions was,
+ // so use the higher, i.e. more conservative, of their entry probabilities.
+ BranchProbability Prob1 = vputils::getRegionEntryProbability(Region1);
+ BranchProbability Prob2 = vputils::getRegionEntryProbability(Region2);
+ if (!Prob1.isUnknown() && !Prob2.isUnknown() && Prob2 < Prob1)
+ Region2->getEntryBranchOnMask()->setMetadata(
+ LLVMContext::MD_prof,
+ Region1->getEntryBranchOnMask()->getMetadata(LLVMContext::MD_prof));
+
// Note: No fusion-preventing memory dependencies are expected in either
// region. Such dependencies should be rejected during earlier dependence
// checks, which guarantee accesses can be re-ordered for vectorization.
@@ -544,6 +553,12 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
PredRecipe->getDebugLoc());
+ // Move the predicated recipes's branch weights onto the guarding
+ // branch-on-mask.
+ if (MDNode *BW = RecipeWithoutMask->getMetadata(LLVMContext::MD_prof)) {
+ BOMRecipe->setMetadata(LLVMContext::MD_prof, BW);
+ RecipeWithoutMask->eraseMetadata(LLVMContext::MD_prof);
+ }
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
auto *Exiting = Plan.createVPBasicBlock(Twine(RegionName) + ".continue");
@@ -3731,6 +3746,9 @@ static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
VPIRMetadata CommonMetadata = *Recipes.front();
for (VPReplicateRecipe *Recipe : drop_begin(Recipes))
CommonMetadata.intersect(*Recipe);
+ // The recipe the common metadata is used for is not predicated, so drop
+ // !prof.
+ CommonMetadata.eraseMetadata(LLVMContext::MD_prof);
return CommonMetadata;
}
@@ -5586,6 +5604,19 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
}
}
+void VPlanTransforms::dropBranchWeightsFromUnguardedRecipes(VPlan &Plan) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getEntry()))) {
+ for (VPRecipeBase &R : *VPBB) {
+ auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+ if (RepR && RepR->isPredicated())
+ continue;
+ if (auto *MD = dyn_cast<VPIRMetadata>(&R))
+ MD->eraseMetadata(LLVMContext::MD_prof);
+ }
+ }
+}
+
/// Returns true if \p Info's parameter kinds are compatible with \p Args.
static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
PredicatedScalarEvolution &PSE, const Loop *L) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index af8f6de26e0f9..708180d8ca146 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -594,6 +594,10 @@ struct VPlanTransforms {
/// enabled by prior widening of consecutive memory operations for now.
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
+ /// Drop the branch weights from all recipes that cannot preserve them.
+ /// Currently that are all recipes, except VPReplicateRecipes.
+ static void dropBranchWeightsFromUnguardedRecipes(VPlan &Plan);
+
/// Convert call VPInstructions in \p Plan into widened call, vector
/// intrinsic or replicate recipes based on a cost comparison via \p CostCtx.
static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index a430de94cca14..57fb7b833d329 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -710,8 +710,11 @@ static void convertRecipesInRegionBlocksToSingleScalar(VPlan &Plan, Type *IdxTy,
RepR->replaceAllUsesWith(NewR);
RepR->eraseFromParent();
} else if (auto *BranchOnMask = dyn_cast<VPBranchOnMaskRecipe>(&OldR)) {
+ // Carry the branch weights describing how often the predicated block
+ // is entered onto the generated conditional branch.
Builder.createNaryOp(VPInstruction::BranchOnCond,
- {BranchOnMask->getOperand(0)}, OldDL);
+ {BranchOnMask->getOperand(0)},
+ /*Inst=*/nullptr, {}, *BranchOnMask, OldDL);
BranchOnMask->eraseFromParent();
} else if (auto *PredPhi = dyn_cast<VPPredInstPHIRecipe>(&OldR)) {
VPValue *PredOp = PredPhi->getOperand(0);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 93b18b31e9e7d..9f2bb3b636198 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -19,6 +19,7 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
using namespace llvm;
@@ -1050,6 +1051,79 @@ SmallVector<VPUser *> vputils::collectUsersRecursively(VPValue *V) {
return Users.takeVector();
}
+/// Returns the probability of the edge from \p Src to \p Dst, taken from the
+/// branch weights recorded on Src's terminator, or unknown if not available.
+/// See llvm::getBranchProbability in llvm/Transforms/Utils/LoopUtils.h for the
+/// IR version.
+static BranchProbability getEdgeProbability(const VPBasicBlock *Src,
+ const VPBasicBlock *Dst) {
+ // With a single successor the edge is always taken.
+ ArrayRef<VPBlockBase *> Successors = Src->getSuccessors();
+ if (Successors.size() == 1)
+ return BranchProbability::getOne();
+
+ auto *Term = dyn_cast_if_present<VPInstruction>(Src->getTerminator());
+ SmallVector<uint32_t> Weights;
+ if (!Term ||
+ !extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) ||
+ Weights.size() != Successors.size())
+ return BranchProbability::getUnknown();
+
+ uint64_t Total = sum_of(Weights, uint64_t(0));
+ if (Total == 0)
+ return BranchProbability::getUnknown();
+
+ // Sum the weights of all edges from Src to Dst; the same block may be the
+ // destination of multiple successors, e.g. for switches.
+ uint64_t ToDst = 0;
+ for (const auto &[Succ, Weight] : zip(Successors, Weights))
+ if (Succ == Dst)
+ ToDst += Weight;
+ return BranchProbability::getBranchProbability(ToDst, Total);
+}
+
+DenseMap<const VPBasicBlock *, BranchProbability>
+vputils::computeBlockProbabilities(ArrayRef<VPBasicBlock *> Blocks) {
+ assert(!Blocks.empty() && "expected at least the header block");
+ DenseMap<const VPBasicBlock *, BranchProbability> Probabilities;
+ // The header (first block) always executes. Any other block executes if any
+ // of its incoming edges is taken, so accumulate their probabilities.
+ Probabilities[Blocks.front()] = BranchProbability::getOne();
+ for (VPBasicBlock *VPBB : Blocks.drop_front()) {
+ BranchProbability Prob = BranchProbability::getZero();
+ // A predecessor may be listed once per edge to VPBB, e.g. for a switch with
+ // multiple cases branching here.
+ SmallSetVector<VPBlockBase *, 4> Preds(from_range, VPBB->getPredecessors());
+ for (VPBasicBlock *PredVPBB : VPBlockUtils::blocksAs<VPBasicBlock>(Preds)) {
+ BranchProbability PredProb = Probabilities.lookup(PredVPBB);
+ BranchProbability EdgeProb = getEdgeProbability(PredVPBB, VPBB);
+ if (PredProb.isUnknown() || EdgeProb.isUnknown()) {
+ Prob = BranchProbability::getUnknown();
+ break;
+ }
+ BranchProbability Contribution = PredProb * EdgeProb;
+ // Force to lowest possible probability if result gets rounded to zero.
+ if (Contribution.isZero() && !PredProb.isZero() && !EdgeProb.isZero())
+ Contribution = BranchProbability::getRaw(1);
+ Prob += Contribution;
+ }
+ Probabilities[VPBB] = Prob;
+ }
+ return Probabilities;
+}
+
+BranchProbability
+vputils::getRegionEntryProbability(const VPRegionBlock *Region) {
+ const VPBranchOnMaskRecipe *Guard = Region->getEntryBranchOnMask();
+ SmallVector<uint32_t, 2> Weights;
+ if (!extractBranchWeights(Guard->getMetadata(LLVMContext::MD_prof), Weights))
+ return BranchProbability::getUnknown();
+ // The weights are {Taken, NotTaken}: the region is entered if the branch on
+ // the guarding mask is taken.
+ return BranchProbability::getBranchProbability(Weights[0],
+ sum_of(Weights, uint64_t(0)));
+}
+
VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
ArrayRef<VPValue *> Operands,
const DataLayout &DL) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..1b3d55e65b4e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -10,6 +10,7 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANUTILS_H
#include "VPlan.h"
+#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
@@ -221,6 +222,18 @@ SmallVector<VPUser *> collectUsersRecursively(VPValue *V);
VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
const DataLayout &DL);
+/// Computes for each block in \p Blocks the probability that it executes,
+/// relative to the first block in \p Blocks (the header block), which always
+/// executes. The probability of a block is the accumulated probability of its
+/// incoming edges.
+DenseMap<const VPBasicBlock *, BranchProbability>
+computeBlockProbabilities(ArrayRef<VPBasicBlock *> Blocks);
+
+/// Returns the probability of entering replicate region \p Region, taken from
+/// the branch weights recorded on its guarding branch-on-mask, or unknown if it
+/// carries none.
+BranchProbability getRegionEntryProbability(const VPRegionBlock *Region);
+
namespace detail {
/// Template-independent implementation for pullOutPermutations.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index a7271524f0191..ce82fb4978cf1 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -25,6 +25,7 @@
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeScalarizationDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeCallWideningDecisions
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dropBranchWeightsFromUnguardedRecipes
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeFindIVReductions
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
index a2448dfc371ca..9e92869f94669 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
@@ -82,11 +82,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; CONSTRUCT-NEXT: Successor(s): if.then
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: if.then:
-; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp>
+; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!prof {1, 3})
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; CONSTRUCT-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp>
+; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!prof {1, 3})
; CONSTRUCT-NEXT: Successor(s): latch
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: latch:
@@ -129,7 +129,7 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; REGION-EMPTY:
; REGION-NEXT: <xVFxUF> pred.store: {
; REGION-NEXT: pred.store.entry:
-; REGION-NEXT: BRANCH-ON-MASK ir<%cmp>
+; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!prof {1, 3})
; REGION-NEXT: Successor(s): pred.store.if, pred.store.continue
; REGION-EMPTY:
; REGION-NEXT: pred.store.if:
@@ -171,7 +171,7 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; DISSOLVE-NEXT: WIDEN-CAST ir<%t> = trunc ir<%add> to i16
; DISSOLVE-NEXT: WIDEN-CAST ir<%ext> = sext ir<%t> to i64
; DISSOLVE-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = extractelement ir<%cmp>, ir<0>
-; DISSOLVE-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; DISSOLVE-NEXT: EMIT branch-on-cond vp<[[VP1]]> (!prof {1, 3})
; DISSOLVE-NEXT: Successor(s): pred.store.if, pred.store.continue
; DISSOLVE-EMPTY:
; DISSOLVE-NEXT: pred.store.if:
@@ -183,7 +183,7 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; DISSOLVE-EMPTY:
; DISSOLVE-NEXT: pred.store.continue:
; DISSOLVE-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = extractelement ir<%cmp>, ir<1>
-; DISSOLVE-NEXT: EMIT branch-on-cond vp<[[VP5]]>
+; DISSOLVE-NEXT: EMIT branch-on-cond vp<[[VP5]]> (!prof {1, 3})
; DISSOLVE-NEXT: Successor(s): pred.store.if, pred.store.continue
; DISSOLVE-EMPTY:
; DISSOLVE-NEXT: pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
index 8a63e40b1ea40..1d84b17d3dd71 100644
--- a/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
@@ -10,24 +10,24 @@ define void @predicated_store(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7:![0-9]+]]
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF1]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -39,24 +39,24 @@ define void @predicated_store(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7:![0-9]+]]
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF1]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
@@ -97,26 +97,26 @@ define void @predicated_store_sunk_address(ptr %a, ptr %b, i32 %n) {
; VF4IC1: br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF7]]:
; VF4IC1: [[PRED_STORE_CONTINUE8]]:
-; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
; VF4IC1: br i1 [[C1:%.*]], label %[[M:.*]], label %[[LATCH:.*]], !prof [[PROF12:![0-9]+]]
; VF4IC1: [[M]]:
-; VF4IC1: br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF7]]
; VF4IC1: [[IF]]:
; VF4IC1: [[LATCH]]:
; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -130,26 +130,26 @@ define void @predicated_store_sunk_address(ptr %a, ptr %b, i32 %n) {
; VF2IC2: br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
+; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF8]]:
; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2: br i1 [[TMP25:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF2IC2: br i1 [[TMP25:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
; VF2IC2: br i1 [[C1:%.*]], label %[[M:.*]], label %[[LATCH:.*]], !prof [[PROF12:![0-9]+]]
; VF2IC2: [[M]]:
-; VF2IC2: br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF7]]
; VF2IC2: [[IF]]:
; VF2IC2: [[LATCH]]:
; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -287,9 +287,9 @@ define void @predicated_store_unprofiled_predicate(ptr %a, i32 %n) {
; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
@@ -316,9 +316,9 @@ define void @predicated_store_unprofiled_predicate(ptr %a, i32 %n) {
; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
@@ -358,27 +358,27 @@ define void @predicated_store_almost_always_taken(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF19:![0-9]+]]
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF20:![0-9]+]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP20:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP21:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_almost_always_taken(
@@ -387,27 +387,27 @@ define void @predicated_store_almost_always_taken(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF19:![0-9]+]]
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF20:![0-9]+]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP20:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP21:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -443,27 +443,27 @@ define void @predicated_store_non_reciprocal(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF22]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF22]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF22]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP21:![0-9]+]]
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF22:![0-9]+]]
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF24:![0-9]+]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP23:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP25:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_non_reciprocal(
@@ -472,27 +472,27 @@ define void @predicated_store_non_reciprocal(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF22]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF22]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF22]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP21:![0-9]+]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF22:![0-9]+]]
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF24:![0-9]+]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP23:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP25:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -528,29 +528,29 @@ define void @predicated_store_taken_weight_underflow(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF26]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF26]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF26]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP14:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP24:![0-9]+]]
+; VF4IC1: br i1 [[TMP14:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[C1:%.*]], label %[[IF_1:.*]], label %[[LATCH:.*]], !prof [[PROF25:![0-9]+]]
+; VF4IC1: br i1 [[C1:%.*]], label %[[IF_1:.*]], label %[[LATCH:.*]], !prof [[PROF28:![0-9]+]]
; VF4IC1: [[IF_1]]:
-; VF4IC1: br i1 [[C2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH]], !prof [[PROF25]]
+; VF4IC1: br i1 [[C2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH]], !prof [[PROF28]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP26:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP29:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_taken_weight_underflow(
@@ -559,29 +559,29 @@ define void @predicated_store_taken_weight_underflow(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF26]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF26]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF26]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP24:![0-9]+]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[C1:%.*]], label %[[IF_1:.*]], label %[[LATCH:.*]], !prof [[PROF25:![0-9]+]]
+; VF2IC2: br i1 [[C1:%.*]], label %[[IF_1:.*]], label %[[LATCH:.*]], !prof [[PROF28:![0-9]+]]
; VF2IC2: [[IF_1]]:
-; VF2IC2: br i1 [[C2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH]], !prof [[PROF25]]
+; VF2IC2: br i1 [[C2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH]], !prof [[PROF28]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP26:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP29:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -618,27 +618,27 @@ define void @predicated_block_with_multiple_ops(ptr noalias %a, ptr noalias %b,
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP27:![0-9]+]]
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP28:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP31:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_block_with_multiple_ops(
@@ -647,27 +647,27 @@ define void @predicated_block_with_multiple_ops(ptr noalias %a, ptr noalias %b,
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP27:![0-9]+]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP28:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP31:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -710,31 +710,31 @@ define void @predicated_store_multiple_incoming_edges(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF32]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF32]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF32]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP29:![0-9]+]]
+; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF6]]
+; VF4IC1: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF4IC1: [[PATH_A]]:
; VF4IC1: br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF4IC1: [[PATH_B]]:
-; VF4IC1: br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]], !prof [[PROF7]]
+; VF4IC1: br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]], !prof [[PROF1]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP30:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP34:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_multiple_incoming_edges(
@@ -743,31 +743,31 @@ define void @predicated_store_multiple_incoming_edges(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF32]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF32]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF32]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP29:![0-9]+]]
+; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF6]]
+; VF2IC2: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF2IC2: [[PATH_A]]:
; VF2IC2: br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF2IC2: [[PATH_B]]:
-; VF2IC2: br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]], !prof [[PROF7]]
+; VF2IC2: br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]], !prof [[PROF1]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP30:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP34:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -822,19 +822,19 @@ define void @predicated_store_multiple_incoming_edges_one_unprofiled(ptr %a, i32
; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP31:![0-9]+]]
+; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF6]]
+; VF4IC1: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF4IC1: [[PATH_A]]:
; VF4IC1: br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF4IC1: [[PATH_B]]:
; VF4IC1: br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]]
; VF4IC1: [[IF_THEN]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP32:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP36:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_multiple_incoming_edges_one_unprofiled(
@@ -855,19 +855,19 @@ define void @predicated_store_multiple_incoming_edges_one_unprofiled(ptr %a, i32
; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP31:![0-9]+]]
+; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF6]]
+; VF2IC2: br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF2IC2: [[PATH_A]]:
; VF2IC2: br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF2IC2: [[PATH_B]]:
; VF2IC2: br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]]
; VF2IC2: [[IF_THEN]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP32:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP36:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -912,40 +912,40 @@ define void @predicated_store_switch_multiple_cases(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF37:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF7]]:
; VF4IC1: [[PRED_STORE_CONTINUE8]]:
-; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]], !prof [[PROF37]]
; VF4IC1: [[PRED_STORE_IF9]]:
; VF4IC1: [[PRED_STORE_CONTINUE10]]:
-; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]], !prof [[PROF37]]
; VF4IC1: [[PRED_STORE_IF11]]:
; VF4IC1: [[PRED_STORE_CONTINUE12]]:
-; VF4IC1: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; VF4IC1: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]], !prof [[PROF37]]
; VF4IC1: [[PRED_STORE_IF13]]:
; VF4IC1: [[PRED_STORE_CONTINUE14]]:
-; VF4IC1: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP33:![0-9]+]]
+; VF4IC1: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: ], !prof [[PROF34:![0-9]+]]
+; VF4IC1: ], !prof [[PROF39:![0-9]+]]
; VF4IC1: [[IF_THEN:.*:]]
; VF4IC1: [[OTHER:.*:]]
; VF4IC1: [[LATCH:.*:]]
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP35:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP40:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_switch_multiple_cases(
@@ -954,40 +954,40 @@ define void @predicated_store_switch_multiple_cases(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
+; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF37:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF8]]:
; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
+; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]], !prof [[PROF37]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]], !prof [[PROF37]]
; VF2IC2: [[PRED_STORE_IF12]]:
; VF2IC2: [[PRED_STORE_CONTINUE13]]:
-; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]], !prof [[PROF37]]
; VF2IC2: [[PRED_STORE_IF14]]:
; VF2IC2: [[PRED_STORE_CONTINUE15]]:
-; VF2IC2: br i1 [[TMP24:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP33:![0-9]+]]
+; VF2IC2: br i1 [[TMP24:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: ], !prof [[PROF34:![0-9]+]]
+; VF2IC2: ], !prof [[PROF39:![0-9]+]]
; VF2IC2: [[IF_THEN:.*:]]
; VF2IC2: [[OTHER:.*:]]
; VF2IC2: [[LATCH:.*:]]
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP35:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP40:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -1030,30 +1030,30 @@ define void @merged_replicate_regions_inconsistent_weights(ptr noalias %a, ptr n
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP16:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP36:![0-9]+]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
-; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
; VF4IC1: [[LOOP:.*]]:
-; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF19]]
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF20]]
; VF4IC1: [[IF_THEN_1]]:
; VF4IC1: [[MERGE]]:
-; VF4IC1: br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF37:![0-9]+]]
+; VF4IC1: br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF42:![0-9]+]]
; VF4IC1: [[IF_THEN_2]]:
; VF4IC1: [[LATCH]]:
-; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP38:![0-9]+]]
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP43:![0-9]+]]
; VF4IC1: [[EXIT]]:
;
; VF2IC2-LABEL: define void @merged_replicate_regions_inconsistent_weights(
@@ -1062,30 +1062,30 @@ define void @merged_replicate_regions_inconsistent_weights(ptr noalias %a, ptr n
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP36:![0-9]+]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
-; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
; VF2IC2: [[LOOP:.*]]:
-; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF19]]
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF20]]
; VF2IC2: [[IF_THEN_1]]:
; VF2IC2: [[MERGE]]:
-; VF2IC2: br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF37:![0-9]+]]
+; VF2IC2: br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF42:![0-9]+]]
; VF2IC2: [[IF_THEN_2]]:
; VF2IC2: [[LATCH]]:
-; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP38:![0-9]+]]
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP43:![0-9]+]]
; VF2IC2: [[EXIT]]:
;
entry:
@@ -1129,82 +1129,92 @@ exit:
!7 = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
;.
; VF4IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
-; VF4IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 249}
-; VF4IC1: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
-; VF4IC1: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
-; VF4IC1: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
-; VF4IC1: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 250}
-; VF4IC1: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
-; VF4IC1: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
+; VF4IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 7}
+; VF4IC1: [[PROF2]] = !{!"branch_weights", i32 1, i32 249}
+; VF4IC1: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; VF4IC1: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
+; VF4IC1: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VF4IC1: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 250}
+; VF4IC1: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
; VF4IC1: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
-; VF4IC1: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META3]], [[META10:![0-9]+]]}
+; VF4IC1: [[LOOP9]] = distinct !{[[LOOP9]], [[META5]], [[META4]], [[META10:![0-9]+]]}
; VF4IC1: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
-; VF4IC1: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]], [[META5]]}
+; VF4IC1: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF12]] = !{!"branch_weights", i32 1, i32 1}
-; VF4IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]], [[META4]]}
-; VF4IC1: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]], [[META3]]}
-; VF4IC1: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP18]] = distinct !{[[LOOP18]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[PROF19]] = !{!"branch_weights", i32 1000, i32 1}
-; VF4IC1: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP21]] = distinct !{[[LOOP21]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[PROF22]] = !{!"branch_weights", i32 3, i32 2}
-; VF4IC1: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP24]] = distinct !{[[LOOP24]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[PROF25]] = !{!"branch_weights", i32 1, i32 100000}
-; VF4IC1: [[LOOP26]] = distinct !{[[LOOP26]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP27]] = distinct !{[[LOOP27]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[LOOP28]] = distinct !{[[LOOP28]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP29]] = distinct !{[[LOOP29]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[LOOP30]] = distinct !{[[LOOP30]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP31]] = distinct !{[[LOOP31]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[LOOP32]] = distinct !{[[LOOP32]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP33]] = distinct !{[[LOOP33]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[PROF34]] = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
-; VF4IC1: [[LOOP35]] = distinct !{[[LOOP35]], [[META4]], [[META3]], [[META10]]}
-; VF4IC1: [[LOOP36]] = distinct !{[[LOOP36]], [[META3]], [[META4]], [[META5]]}
-; VF4IC1: [[PROF37]] = !{!"branch_weights", i32 1, i32 1000}
-; VF4IC1: [[LOOP38]] = distinct !{[[LOOP38]], [[META4]], [[META3]], [[META10]]}
+; VF4IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META10]]}
+; VF4IC1: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]], [[META5]]}
+; VF4IC1: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META4]]}
+; VF4IC1: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[PROF18]] = !{!"branch_weights", i32 1072669155, i32 1072669}
+; VF4IC1: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[PROF20]] = !{!"branch_weights", i32 1000, i32 1}
+; VF4IC1: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[PROF22]] = !{!"branch_weights", i32 1288490189, i32 858993459}
+; VF4IC1: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[PROF24]] = !{!"branch_weights", i32 3, i32 2}
+; VF4IC1: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[PROF26]] = !{!"branch_weights", i32 1, i32 2147483647}
+; VF4IC1: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[PROF28]] = !{!"branch_weights", i32 1, i32 100000}
+; VF4IC1: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[LOOP30]] = distinct !{[[LOOP30]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[PROF32]] = !{!"branch_weights", i32 7, i32 25}
+; VF4IC1: [[LOOP33]] = distinct !{[[LOOP33]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[LOOP34]] = distinct !{[[LOOP34]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[LOOP35]] = distinct !{[[LOOP35]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[LOOP36]] = distinct !{[[LOOP36]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[PROF37]] = !{!"branch_weights", i32 3, i32 5}
+; VF4IC1: [[LOOP38]] = distinct !{[[LOOP38]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[PROF39]] = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
+; VF4IC1: [[LOOP40]] = distinct !{[[LOOP40]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[LOOP41]] = distinct !{[[LOOP41]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[PROF42]] = !{!"branch_weights", i32 1, i32 1000}
+; VF4IC1: [[LOOP43]] = distinct !{[[LOOP43]], [[META5]], [[META4]], [[META10]]}
;.
; VF2IC2: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
-; VF2IC2: [[PROF1]] = !{!"branch_weights", i32 1, i32 249}
-; VF2IC2: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
-; VF2IC2: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
-; VF2IC2: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
-; VF2IC2: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 250}
-; VF2IC2: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
-; VF2IC2: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
+; VF2IC2: [[PROF1]] = !{!"branch_weights", i32 1, i32 7}
+; VF2IC2: [[PROF2]] = !{!"branch_weights", i32 1, i32 249}
+; VF2IC2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; VF2IC2: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
+; VF2IC2: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VF2IC2: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 250}
+; VF2IC2: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
; VF2IC2: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
-; VF2IC2: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META3]], [[META10:![0-9]+]]}
+; VF2IC2: [[LOOP9]] = distinct !{[[LOOP9]], [[META5]], [[META4]], [[META10:![0-9]+]]}
; VF2IC2: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
-; VF2IC2: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]], [[META5]]}
+; VF2IC2: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF12]] = !{!"branch_weights", i32 1, i32 1}
-; VF2IC2: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]], [[META4]]}
-; VF2IC2: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]], [[META3]]}
-; VF2IC2: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP18]] = distinct !{[[LOOP18]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[PROF19]] = !{!"branch_weights", i32 1000, i32 1}
-; VF2IC2: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP21]] = distinct !{[[LOOP21]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[PROF22]] = !{!"branch_weights", i32 3, i32 2}
-; VF2IC2: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP24]] = distinct !{[[LOOP24]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[PROF25]] = !{!"branch_weights", i32 1, i32 100000}
-; VF2IC2: [[LOOP26]] = distinct !{[[LOOP26]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP27]] = distinct !{[[LOOP27]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[LOOP28]] = distinct !{[[LOOP28]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP29]] = distinct !{[[LOOP29]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[LOOP30]] = distinct !{[[LOOP30]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP31]] = distinct !{[[LOOP31]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[LOOP32]] = distinct !{[[LOOP32]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP33]] = distinct !{[[LOOP33]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[PROF34]] = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
-; VF2IC2: [[LOOP35]] = distinct !{[[LOOP35]], [[META4]], [[META3]], [[META10]]}
-; VF2IC2: [[LOOP36]] = distinct !{[[LOOP36]], [[META3]], [[META4]], [[META5]]}
-; VF2IC2: [[PROF37]] = !{!"branch_weights", i32 1, i32 1000}
-; VF2IC2: [[LOOP38]] = distinct !{[[LOOP38]], [[META4]], [[META3]], [[META10]]}
+; VF2IC2: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META10]]}
+; VF2IC2: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]], [[META5]]}
+; VF2IC2: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META4]]}
+; VF2IC2: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[PROF18]] = !{!"branch_weights", i32 1072669155, i32 1072669}
+; VF2IC2: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[PROF20]] = !{!"branch_weights", i32 1000, i32 1}
+; VF2IC2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[PROF22]] = !{!"branch_weights", i32 1288490189, i32 858993459}
+; VF2IC2: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[PROF24]] = !{!"branch_weights", i32 3, i32 2}
+; VF2IC2: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[PROF26]] = !{!"branch_weights", i32 1, i32 2147483647}
+; VF2IC2: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[PROF28]] = !{!"branch_weights", i32 1, i32 100000}
+; VF2IC2: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[LOOP30]] = distinct !{[[LOOP30]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[PROF32]] = !{!"branch_weights", i32 7, i32 25}
+; VF2IC2: [[LOOP33]] = distinct !{[[LOOP33]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[LOOP34]] = distinct !{[[LOOP34]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[LOOP35]] = distinct !{[[LOOP35]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[LOOP36]] = distinct !{[[LOOP36]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[PROF37]] = !{!"branch_weights", i32 3, i32 5}
+; VF2IC2: [[LOOP38]] = distinct !{[[LOOP38]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[PROF39]] = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
+; VF2IC2: [[LOOP40]] = distinct !{[[LOOP40]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[LOOP41]] = distinct !{[[LOOP41]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[PROF42]] = !{!"branch_weights", i32 1, i32 1000}
+; VF2IC2: [[LOOP43]] = distinct !{[[LOOP43]], [[META5]], [[META4]], [[META10]]}
;.
>From 4828188b8118f1ba7bc5a80ae941c6feb119e7eb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 3 Aug 2026 20:03:19 +0100
Subject: [PATCH 2/7] !fixup verify against IR BFI, profcheck
step
---
.../Transforms/Vectorize/LoopVectorize.cpp | 64 ++++
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
llvm/test/lit.cfg.py | 2 -
llvm/utils/profcheck-xfail.txt | 348 ++++++++++++++++++
4 files changed, 413 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 93cfaaacc9c27..1ab721368b9b1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6537,6 +6537,66 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
// optimizations.
static void printOptimizedVPlan(VPlan &) {}
+#ifndef NDEBUG
+/// Cross-check the probabilities vputils::computeBlockProbabilities computes
+/// for the blocks of the loop region of \p Plan against the frequencies \p BFI
+/// computed for the corresponding blocks of \p OrigLoop.
+/// FIXME: Temporary verification aid, to be removed.
+static bool verifyBlockProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
+ LoopInfo *LI,
+ BlockFrequencyInfo &BFI) {
+
+ // The verification is limited inner loops where the latch is the only exiting
+ // block and there are no extra VPBBs not mapped to IR BBs (when tailfolding).
+ if (Plan.isOuterLoop() || OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch() || Plan.hasTailFolded())
+ return true;
+
+ // Visit the blocks of the loop region in the same order as
+ // introduceMasksAndLinearize does.
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock());
+ auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
+ assert(Blocks.size() == OrigLoop->getNumBlocks());
+
+ LoopBlocksRPO OrigRPO(OrigLoop);
+ OrigRPO.perform(LI);
+
+ uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
+ if (HeaderFreq == 0)
+ return true;
+
+ // BFI's fixed-point mass propagation rounds per edge, losing up to 1 ULP per
+ // block on the path from the header.
+ uint64_t Tolerance =
+ Blocks.size() + BranchProbability::getDenominator() / HeaderFreq;
+
+ DenseMap<const VPBasicBlock *, BranchProbability> Probabilities =
+ vputils::computeBlockProbabilities(Blocks);
+ for (const auto &[VPBB, BB] :
+ zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
+ BranchProbability Computed = Probabilities.lookup(VPBB);
+ // Currently VPlan-based probabilities are only computed when all blocks
+ // have branch-weights.
+ if (Computed.isUnknown())
+ continue;
+
+ // Clamp the frequency to the header's; it may exceed it slightly due to
+ // BFI's rounding.
+ uint64_t Freq = BFI.getBlockFreq(BB).getFrequency();
+ BranchProbability Expected = BranchProbability::getBranchProbability(
+ std::min(Freq, HeaderFreq), HeaderFreq);
+ if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
+ Tolerance)
+ continue;
+
+ errs() << "Block probability mismatch for " << VPBB->getName() << ": VPlan "
+ << Computed << ", BlockFrequencyInfo " << Expected << "\n";
+ return false;
+ }
+ return true;
+}
+#endif
+
VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
bool IsInnerLoop = OrigLoop->isInnermost();
@@ -6618,6 +6678,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
if (CM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
+
+
+ assert(verifyBlockProbabilitiesMatchBFI(*VPlan0, OrigLoop, LI, CM.getBFI()) &&
+ "block probabilities do not match the original loop's frequencies");
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 708180d8ca146..a7ad2235a5b90 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -595,7 +595,7 @@ struct VPlanTransforms {
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
/// Drop the branch weights from all recipes that cannot preserve them.
- /// Currently that are all recipes, except VPReplicateRecipes.
+ /// Currently that is all recipes, except VPReplicateRecipes.
static void dropBranchWeightsFromUnguardedRecipes(VPlan &Plan);
/// Convert call VPInstructions in \p Plan into widened call, vector
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 6f04a855d8cb8..8a6e110f968b3 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -50,8 +50,6 @@
# Exclude llvm-objcopy tests - not the target of this effort, and some use
# cat in ways that conflict with how profcheck uses it.
config.excludes.append("llvm-objcopy")
- # (Issue #161235) Temporarily exclude LoopVectorize.
- config.excludes.append("LoopVectorize")
# Exclude suites that fail due to inserted profile annotations.
config.excludes.extend(["UpdateTestChecks", "Bitcode"])
# TODO(#166655): Reenable Instrumentation tests
diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt
index 0e90386d0ffd3..78484139d784a 100644
--- a/llvm/utils/profcheck-xfail.txt
+++ b/llvm/utils/profcheck-xfail.txt
@@ -87,6 +87,354 @@ Transforms/InstCombine/xor-and-or.ll
Transforms/InstCombine/zext-bool-add-sub.ll
Transforms/LoopIdiom/AArch64/byte-compare-index.ll
Transforms/LoopIdiom/AArch64/find-first-byte.ll
+Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
+Transforms/LoopVectorize/AArch64/arith-costs.ll
+Transforms/LoopVectorize/AArch64/blend-costs.ll
+Transforms/LoopVectorize/AArch64/bounded-load.ll
+Transforms/LoopVectorize/AArch64/check-prof-info.ll
+Transforms/LoopVectorize/AArch64/cmp_cost.ll
+Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll
+Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
+Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
+Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+Transforms/LoopVectorize/AArch64/early_exit_costs.ll
+Transforms/LoopVectorize/AArch64/epilog-iv-live-outs.ll
+Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
+Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
+Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
+Transforms/LoopVectorize/AArch64/extend-vectorization-factor-for-unprofitable-memops.ll
+Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll
+Transforms/LoopVectorize/AArch64/findlast-epilogue-loop.ll
+Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
+Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
+Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+Transforms/LoopVectorize/AArch64/gather-cost.ll
+Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+Transforms/LoopVectorize/AArch64/induction-costs.ll
+Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
+Transforms/LoopVectorize/AArch64/interleave_count_for_estimated_tc.ll
+Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
+Transforms/LoopVectorize/AArch64/intrinsiccost.ll
+Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+Transforms/LoopVectorize/AArch64/load-cast-context.ll
+Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+Transforms/LoopVectorize/AArch64/masked-call.ll
+Transforms/LoopVectorize/AArch64/neon-gather-cost.ll
+Transforms/LoopVectorize/AArch64/neon-inloop-reductions.ll
+Transforms/LoopVectorize/AArch64/neoverse-epilogue-vect.ll
+Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-extends-shared-with-reduce.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll
+Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+Transforms/LoopVectorize/AArch64/predicated-costs.ll
+Transforms/LoopVectorize/AArch64/predication_costs.ll
+Transforms/LoopVectorize/AArch64/reduction-cost.ll
+Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+Transforms/LoopVectorize/AArch64/reduction-small-size.ll
+Transforms/LoopVectorize/AArch64/reg-usage.ll
+Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
+Transforms/LoopVectorize/AArch64/scalable-reductions.ll
+Transforms/LoopVectorize/AArch64/sdiv-pow2.ll
+Transforms/LoopVectorize/AArch64/select-costs.ll
+Transforms/LoopVectorize/AArch64/select-index.ll
+Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
+Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
+Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+Transforms/LoopVectorize/AArch64/strict-fadd.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
+Transforms/LoopVectorize/AArch64/sve-gather-scatter-cost.ll
+Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
+Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
+Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+Transforms/LoopVectorize/AArch64/sve-large-strides.ll
+Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+Transforms/LoopVectorize/AArch64/sve-select-cmp.ll
+Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-fold-tail.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
+Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+Transforms/LoopVectorize/AArch64/vector-reverse.ll
+Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+Transforms/LoopVectorize/ARM/mve-multiexit.ll
+Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
+Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
+Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
+Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll
+Transforms/LoopVectorize/SystemZ/pr47665.ll
+Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
+Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll
+Transforms/LoopVectorize/VE/disable_lv.ll
+Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+Transforms/LoopVectorize/VPlan/constant-fold.ll
+Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+Transforms/LoopVectorize/VPlan/find-last.ll
+Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
+Transforms/LoopVectorize/VPlan/interleave-and-scalarize-only.ll
+Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
+Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
+Transforms/LoopVectorize/VPlan/tail-folding.ll
+Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
+Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
+Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
+Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+Transforms/LoopVectorize/X86/CostModel/handle-iptr-with-data-layout-to-not-assert.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+Transforms/LoopVectorize/X86/avx1.ll
+Transforms/LoopVectorize/X86/avx512.ll
+Transforms/LoopVectorize/X86/cast-costs.ll
+Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
+Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+Transforms/LoopVectorize/X86/conversion-cost.ll
+Transforms/LoopVectorize/X86/cost-any-of.ll
+Transforms/LoopVectorize/X86/cost-conditional-branches.ll
+Transforms/LoopVectorize/X86/cost-model-i386.ll
+Transforms/LoopVectorize/X86/cost-model.ll
+Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
+Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
+Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll
+Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+Transforms/LoopVectorize/X86/float-induction-x86.ll
+Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll
+Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll
+Transforms/LoopVectorize/X86/gather_scatter.ll
+Transforms/LoopVectorize/X86/idiv-by-const.ll
+Transforms/LoopVectorize/X86/induction-costs.ll
+Transforms/LoopVectorize/X86/interleaved-accesses-use-after-free.ll
+Transforms/LoopVectorize/X86/intrinsiccost.ll
+Transforms/LoopVectorize/X86/invariant-load-gather.ll
+Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+Transforms/LoopVectorize/X86/iv-live-outs.ll
+Transforms/LoopVectorize/X86/masked-store-cost.ll
+Transforms/LoopVectorize/X86/masked_load_store.ll
+Transforms/LoopVectorize/X86/multi-exit-cost.ll
+Transforms/LoopVectorize/X86/optsize.ll
+Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+Transforms/LoopVectorize/X86/pr23997.ll
+Transforms/LoopVectorize/X86/pr47437.ll
+Transforms/LoopVectorize/X86/pr54634.ll
+Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
+Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+Transforms/LoopVectorize/X86/propagate-metadata.ll
+Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+Transforms/LoopVectorize/X86/scatter_crash.ll
+Transforms/LoopVectorize/X86/small-size.ll
+Transforms/LoopVectorize/X86/strided_load_cost.ll
+Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-live-outs.ll
+Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+Transforms/LoopVectorize/X86/uniformshift.ll
+Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
+Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
+Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
+Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
+Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+Transforms/LoopVectorize/X86/x86-predication.ll
+Transforms/LoopVectorize/alias-mask-needs-freeze.ll
+Transforms/LoopVectorize/alias-mask.ll
+Transforms/LoopVectorize/as_cast.ll
+Transforms/LoopVectorize/bounded-load-multi-exit.ll
+Transforms/LoopVectorize/cast-costs.ll
+Transforms/LoopVectorize/cast-induction.ll
+Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll
+Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+Transforms/LoopVectorize/constant-fold-commutative-and.ll
+Transforms/LoopVectorize/constantfolder.ll
+Transforms/LoopVectorize/cse-casts.ll
+Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll
+Transforms/LoopVectorize/early-exit-calls.ll
+Transforms/LoopVectorize/early-exit-load-live-out.ll
+Transforms/LoopVectorize/early-exit-minmax-trip-count.ll
+Transforms/LoopVectorize/early-exit-unary-ops.ll
+Transforms/LoopVectorize/early_exit_legality.ll
+Transforms/LoopVectorize/early_exit_store_legality.ll
+Transforms/LoopVectorize/early_exit_with_outer_loop.ll
+Transforms/LoopVectorize/epilog-iv-select-cmp.ll
+Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+Transforms/LoopVectorize/epilog-vectorization-dead-epilogue.ll
+Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
+Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
+Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+Transforms/LoopVectorize/epilog-vectorization-scev-expansion.ll
+Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
+Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
+Transforms/LoopVectorize/find-last-iv-interleave.ll
+Transforms/LoopVectorize/find-last-iv-sinkable-expr-epilogue.ll
+Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
+Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+Transforms/LoopVectorize/find-last-iv-sinkable-load.ll
+Transforms/LoopVectorize/find-last.ll
+Transforms/LoopVectorize/first-order-recurrence-complex.ll
+Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
+Transforms/LoopVectorize/first-order-recurrence.ll
+Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+Transforms/LoopVectorize/if-conversion.ll
+Transforms/LoopVectorize/if-pred-non-void.ll
+Transforms/LoopVectorize/if-pred-stores.ll
+Transforms/LoopVectorize/induction.ll
+Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+Transforms/LoopVectorize/interleaved-accesses-2.ll
+Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
+Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
+Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
+Transforms/LoopVectorize/interleaved-accesses.ll
+Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+Transforms/LoopVectorize/iv-select-cmp-nested-loop.ll
+Transforms/LoopVectorize/iv-select-cmp-no-wrap.ll
+Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
+Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+Transforms/LoopVectorize/iv-select-cmp.ll
+Transforms/LoopVectorize/lcssa-crashes.ll
+Transforms/LoopVectorize/loop-form.ll
+Transforms/LoopVectorize/loop-with-constant-exit-condition.ll
+Transforms/LoopVectorize/memdep-fold-tail.ll
+Transforms/LoopVectorize/minmax_reduction.ll
+Transforms/LoopVectorize/multi_early_exit.ll
+Transforms/LoopVectorize/multiple-argmin-argmax.ll
+Transforms/LoopVectorize/multiple-early-exits.ll
+Transforms/LoopVectorize/multiple-exits-versioning.ll
+Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
+Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
+Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
+Transforms/LoopVectorize/optimal-epilog-vectorization.ll
+Transforms/LoopVectorize/optsize.ll
+Transforms/LoopVectorize/phi-with-fastflags.ll
+Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
+Transforms/LoopVectorize/pr32859.ll
+Transforms/LoopVectorize/pr33706.ll
+Transforms/LoopVectorize/pr34681.ll
+Transforms/LoopVectorize/pr43166-fold-tail-by-masking.ll
+Transforms/LoopVectorize/pr44488-predication.ll
+Transforms/LoopVectorize/pr45525.ll
+Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
+Transforms/LoopVectorize/pr55100-expand-scev-predicate-used.ll
+Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
+Transforms/LoopVectorize/predicated-early-exits-interleave.ll
+Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
+Transforms/LoopVectorize/predicated-multiple-exits.ll
+Transforms/LoopVectorize/predicated-single-exit.ll
+Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+Transforms/LoopVectorize/predicator.ll
+Transforms/LoopVectorize/reduction-inloop-uf4.ll
+Transforms/LoopVectorize/reduction-minmax-users-and-predicated.ll
+Transforms/LoopVectorize/reduction-small-size.ll
+Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
+Transforms/LoopVectorize/scalar-select.ll
+Transforms/LoopVectorize/scalarize-masked-call.ll
+Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+Transforms/LoopVectorize/scev-predicate-reasoning.ll
+Transforms/LoopVectorize/select-cmp-blend-chain.ll
+Transforms/LoopVectorize/select-cmp-predicated.ll
+Transforms/LoopVectorize/select-cmp.ll
+Transforms/LoopVectorize/select-index-interleaving.ll
+Transforms/LoopVectorize/select-min-max-last-index-epilogue.ll
+Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll
+Transforms/LoopVectorize/select-reduction.ll
+Transforms/LoopVectorize/select-smax-last-index.ll
+Transforms/LoopVectorize/select-smin-first-index.ll
+Transforms/LoopVectorize/select-smin-last-index.ll
+Transforms/LoopVectorize/select-umax-last-index.ll
+Transforms/LoopVectorize/select-umin-first-index.ll
+Transforms/LoopVectorize/select-umin-last-index.ll
+Transforms/LoopVectorize/single-early-exit-cond-poison.ll
+Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
+Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
+Transforms/LoopVectorize/single-early-exit-interleave-only.ll
+Transforms/LoopVectorize/single-early-exit-interleave.ll
+Transforms/LoopVectorize/single_early_exit.ll
+Transforms/LoopVectorize/single_early_exit_live_outs.ll
+Transforms/LoopVectorize/sink-to-early-exit.ll
+Transforms/LoopVectorize/smax-idx.ll
+Transforms/LoopVectorize/strict-fadd-interleave-only.ll
+Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
+Transforms/LoopVectorize/tail-folding-counting-down.ll
+Transforms/LoopVectorize/tail-folding-masked-mem-opts.ll
+Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll
+Transforms/LoopVectorize/tail-folding-replicate-region.ll
+Transforms/LoopVectorize/tail-folding-switch.ll
+Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+Transforms/LoopVectorize/uniform-blend.ll
+Transforms/LoopVectorize/use-scalar-epilogue-if-tp-fails.ll
+Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll
+Transforms/LoopVectorize/vect.stats.ll
+Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
+Transforms/LoopVectorize/vector-loop-backedge-elimination-predicated-early-exit.ll
+Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
+Transforms/LoopVectorize/version-stride-with-integer-casts.ll
+Transforms/LoopVectorize/vplan-based-stride-mv.ll
Transforms/LowerAtomic/atomic-load.ll
Transforms/LowerAtomic/atomic-swap.ll
Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
>From a9ad032f2f0b2051b9e373ae3d3f5fc509112306 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 8 Aug 2026 07:57:06 +0100
Subject: [PATCH 3/7] !fixup add BFI & VPlan printing test
---
.../branch-weights-match-block-frequencies.ll | 317 ++++++++++++++++
llvm/test/lit.cfg.py | 3 +
llvm/utils/profcheck-xfail.txt | 348 ------------------
3 files changed, 320 insertions(+), 348 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
new file mode 100644
index 0000000000000..0cd3385a11d1d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
@@ -0,0 +1,317 @@
+; RUN: opt -passes='print<block-freq>' -disable-output %s 2>&1 \
+; RUN: | FileCheck --check-prefix=BFI %s
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 \
+; RUN: -vplan-print-after=introduceMasksAndLinearize -disable-output %s 2>&1 \
+; RUN: | FileCheck --check-prefix=VPLAN %s
+
+; Check that the branch weights VPlan puts on the masked recipes of a block
+; describe the same execution probability BlockFrequencyInfo computes for the
+; corresponding block of the original scalar loop.
+
+define void @single_pred(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; Execution probability of each block of the loop
+;
+; %loop 1000/1000 = 1
+; %if.then 250/1000 = 1/4
+; %latch 1000/1000 = 1
+;
+; BFI-LABEL: block-frequency-info: single_pred
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - if.then: float = 250.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'single_pred'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>{{$}}
+; VPLAN-NEXT: EMIT ir<%c.0> = icmp sgt ir<%i>, ir<0>
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %i, ptr %gep.b, align 4
+ %c.0 = icmp sgt i32 %i, 0
+ br i1 %c.0, label %if.then, label %latch, !prof !0
+
+if.then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %i, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @two_preds(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %idx) {
+; Execution probability of each block of the loop. %merge is reached from both
+; %then (1/4) and %else (3/4 * 1/3 = 1/4).
+;
+; %loop 1000/1000 = 1
+; %then 250/1000 = 1/4
+; %else 750/1000 = 3/4
+; %merge 500/1000 = 1/2
+; %latch 1000/1000 = 1
+;
+; BFI-LABEL: block-frequency-info: two_preds
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - then: float = 250.0,
+; BFI-NEXT: - else: float = 750.0,
+; BFI-NEXT: - merge: float = 500.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'two_preds'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN-NEXT: EMIT ir<%c.0> = icmp sgt ir<%i>, ir<0>
+; VPLAN-NEXT: Successor(s): else
+; VPLAN-EMPTY:
+; VPLAN-NEXT: else:
+; VPLAN-NEXT: EMIT vp<[[NOT_C0:%.+]]> = not ir<%c.0>
+; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!prof {3, 1})
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!prof {3, 1})
+; VPLAN-NEXT: Successor(s): then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: then:
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: Successor(s): merge
+; VPLAN-EMPTY:
+; VPLAN-NEXT: merge:
+; VPLAN-NEXT: EMIT vp<[[AND:%.+]]> = logical-and vp<[[NOT_C0]]>, ir<%c.1>
+; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = or vp<[[AND]]>, ir<%c.0>
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!prof {1, 1})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c.0 = icmp sgt i32 %i, 0
+ br i1 %c.0, label %then, label %else, !prof !0
+
+then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %i, ptr %gep.a, align 4
+ br label %merge
+
+else:
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %i, ptr %gep.c, align 4
+ %c.1 = icmp slt i32 %i, -100
+ br i1 %c.1, label %merge, label %latch, !prof !1
+
+merge:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %i, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @nested_ifs(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; Execution probability of each block of the loop
+;
+; %loop 1000/1000 = 1
+; %if.0 250/1000 = 1/4
+; %if.1 125/1000 = 1/8
+; %latch 1000/1000 = 1
+;
+; BFI-LABEL: block-frequency-info: nested_ifs
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - if.0: float = 250.0,
+; BFI-NEXT: - if.1: float = 125.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'nested_ifs'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%i> = load ir<%gep.idx>
+; VPLAN-NEXT: EMIT ir<%c.0> = icmp sgt ir<%i>, ir<0>
+; VPLAN-NEXT: Successor(s): if.0
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.0:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: Successor(s): if.1
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.1:
+; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = logical-and ir<%c.0>, ir<%c.1>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!prof {1, 7})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c.0 = icmp sgt i32 %i, 0
+ br i1 %c.0, label %if.0, label %latch, !prof !0
+
+if.0:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %i, ptr %gep.b, align 4
+ %c.1 = icmp slt i32 %i, 100
+ br i1 %c.1, label %if.1, label %latch, !prof !2
+
+if.1:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %i, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @switch_common_dest(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %idx) {
+; Execution probability of each block of the loop. %if.then is reached from 2 of
+; the switch's cases (125 + 250 = 375) and %default via the default edge (500).
+;
+; %loop 1000/1000 = 1
+; %if.then 375/1000 = 3/8
+; %other 125/1000 = 1/8
+; %default 500/1000 = 1/2
+; %latch 1000/1000 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_common_dest
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - default: float = 500.0,
+; BFI-NEXT: - if.then: float = 375.0,
+; BFI-NEXT: - other: float = 125.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_common_dest'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep.idx>
+; VPLAN-NEXT: Successor(s): other
+; VPLAN-EMPTY:
+; VPLAN-NEXT: other:
+; VPLAN-NEXT: EMIT vp<[[C0:%.+]]> = icmp eq ir<%l>, ir<0>
+; VPLAN-NEXT: EMIT vp<[[C1:%.+]]> = icmp eq ir<%l>, ir<1>
+; VPLAN-NEXT: EMIT vp<[[C2:%.+]]> = icmp eq ir<%l>, ir<2>
+; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
+; VPLAN-NEXT: EMIT vp<[[ANY:%.+]]> = or vp<[[C0_OR_C1]]>, vp<[[C2]]>
+; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[ANY]]>
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!prof {1, 7})
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!prof {3, 5})
+; VPLAN-NEXT: Successor(s): default
+; VPLAN-EMPTY:
+; VPLAN-NEXT: default:
+; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!prof {1, 1})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i8, ptr %idx, i64 %iv
+ %l = load i8, ptr %gep.idx, align 1
+ switch i8 %l, label %default [
+ i8 0, label %if.then
+ i8 1, label %if.then
+ i8 2, label %other
+ ], !prof !4
+
+default:
+ %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
+ store i8 0, ptr %gep.c, align 1
+ br label %latch
+
+if.then:
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ store i8 1, ptr %gep.a, align 1
+ br label %latch
+
+other:
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ store i8 2, ptr %gep.b, align 1
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 3}
+!1 = !{!"branch_weights", i32 1, i32 2}
+!2 = !{!"branch_weights", i32 1, i32 1}
+!3 = !{!"branch_weights", i32 1, i32 999}
+!4 = !{!"branch_weights", i32 500, i32 125, i32 250, i32 125}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 8a6e110f968b3..c68b6f8a7a0ae 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -50,6 +50,9 @@
# Exclude llvm-objcopy tests - not the target of this effort, and some use
# cat in ways that conflict with how profcheck uses it.
config.excludes.append("llvm-objcopy")
+ config.excludes.append("llvm-objcopy")
+ # (Issue #161235) Temporarily exclude LoopVectorize.
+ config.excludes.append("LoopVectorize")
# Exclude suites that fail due to inserted profile annotations.
config.excludes.extend(["UpdateTestChecks", "Bitcode"])
# TODO(#166655): Reenable Instrumentation tests
diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt
index 78484139d784a..0e90386d0ffd3 100644
--- a/llvm/utils/profcheck-xfail.txt
+++ b/llvm/utils/profcheck-xfail.txt
@@ -87,354 +87,6 @@ Transforms/InstCombine/xor-and-or.ll
Transforms/InstCombine/zext-bool-add-sub.ll
Transforms/LoopIdiom/AArch64/byte-compare-index.ll
Transforms/LoopIdiom/AArch64/find-first-byte.ll
-Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll
-Transforms/LoopVectorize/AArch64/arith-costs.ll
-Transforms/LoopVectorize/AArch64/blend-costs.ll
-Transforms/LoopVectorize/AArch64/bounded-load.ll
-Transforms/LoopVectorize/AArch64/check-prof-info.ll
-Transforms/LoopVectorize/AArch64/cmp_cost.ll
-Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
-Transforms/LoopVectorize/AArch64/conditional-scalar-assignment-fold-tail.ll
-Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll
-Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
-Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
-Transforms/LoopVectorize/AArch64/early_exit_costs.ll
-Transforms/LoopVectorize/AArch64/epilog-iv-live-outs.ll
-Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
-Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
-Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
-Transforms/LoopVectorize/AArch64/epilogue-vectorization-fix-scalar-resume-values.ll
-Transforms/LoopVectorize/AArch64/extend-vectorization-factor-for-unprofitable-memops.ll
-Transforms/LoopVectorize/AArch64/find-last-iv-sinkable-expr-epilogue.ll
-Transforms/LoopVectorize/AArch64/findlast-epilogue-loop.ll
-Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
-Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
-Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
-Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
-Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
-Transforms/LoopVectorize/AArch64/gather-cost.ll
-Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
-Transforms/LoopVectorize/AArch64/induction-costs.ll
-Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
-Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
-Transforms/LoopVectorize/AArch64/interleave_count_for_estimated_tc.ll
-Transforms/LoopVectorize/AArch64/interleaved_cost.ll
-Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
-Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
-Transforms/LoopVectorize/AArch64/intrinsiccost.ll
-Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
-Transforms/LoopVectorize/AArch64/load-cast-context.ll
-Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
-Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
-Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
-Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
-Transforms/LoopVectorize/AArch64/masked-call.ll
-Transforms/LoopVectorize/AArch64/neon-gather-cost.ll
-Transforms/LoopVectorize/AArch64/neon-inloop-reductions.ll
-Transforms/LoopVectorize/AArch64/neoverse-epilogue-vect.ll
-Transforms/LoopVectorize/AArch64/optsize_minsize.ll
-Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
-Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
-Transforms/LoopVectorize/AArch64/partial-reduce-extends-shared-with-reduce.ll
-Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
-Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll
-Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
-Transforms/LoopVectorize/AArch64/predicated-costs.ll
-Transforms/LoopVectorize/AArch64/predication_costs.ll
-Transforms/LoopVectorize/AArch64/reduction-cost.ll
-Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
-Transforms/LoopVectorize/AArch64/reduction-small-size.ll
-Transforms/LoopVectorize/AArch64/reg-usage.ll
-Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
-Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
-Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
-Transforms/LoopVectorize/AArch64/scalable-reductions.ll
-Transforms/LoopVectorize/AArch64/sdiv-pow2.ll
-Transforms/LoopVectorize/AArch64/select-costs.ll
-Transforms/LoopVectorize/AArch64/select-index.ll
-Transforms/LoopVectorize/AArch64/simple_early_exit.ll
-Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
-Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
-Transforms/LoopVectorize/AArch64/store-costs-sve.ll
-Transforms/LoopVectorize/AArch64/strict-fadd.ll
-Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
-Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
-Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
-Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
-Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
-Transforms/LoopVectorize/AArch64/sve-epilog-vscale-fixed.ll
-Transforms/LoopVectorize/AArch64/sve-gather-scatter-cost.ll
-Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
-Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
-Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
-Transforms/LoopVectorize/AArch64/sve-large-strides.ll
-Transforms/LoopVectorize/AArch64/sve-multiexit.ll
-Transforms/LoopVectorize/AArch64/sve-select-cmp.ll
-Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
-Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-fold-tail.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-group-requires-scalar-epilogue.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
-Transforms/LoopVectorize/AArch64/transform-narrow-interleave-vscale-x-UF-step.ll
-Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
-Transforms/LoopVectorize/AArch64/vector-reverse.ll
-Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
-Transforms/LoopVectorize/ARM/mve-multiexit.ll
-Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
-Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
-Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
-Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll
-Transforms/LoopVectorize/SystemZ/pr47665.ll
-Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
-Transforms/LoopVectorize/SystemZ/vectorized-epilogue-loop.ll
-Transforms/LoopVectorize/VE/disable_lv.ll
-Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
-Transforms/LoopVectorize/VPlan/constant-fold.ll
-Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
-Transforms/LoopVectorize/VPlan/find-last.ll
-Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
-Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
-Transforms/LoopVectorize/VPlan/interleave-and-scalarize-only.ll
-Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
-Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
-Transforms/LoopVectorize/VPlan/tail-folding.ll
-Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
-Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
-Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
-Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
-Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
-Transforms/LoopVectorize/X86/CostModel/handle-iptr-with-data-layout-to-not-assert.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-6.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-6.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-6.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-7.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-6.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
-Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
-Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
-Transforms/LoopVectorize/X86/avx1.ll
-Transforms/LoopVectorize/X86/avx512.ll
-Transforms/LoopVectorize/X86/cast-costs.ll
-Transforms/LoopVectorize/X86/conditional-scalar-assignment.ll
-Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
-Transforms/LoopVectorize/X86/conversion-cost.ll
-Transforms/LoopVectorize/X86/cost-any-of.ll
-Transforms/LoopVectorize/X86/cost-conditional-branches.ll
-Transforms/LoopVectorize/X86/cost-model-i386.ll
-Transforms/LoopVectorize/X86/cost-model.ll
-Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
-Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
-Transforms/LoopVectorize/X86/epilog-vectorization-ordered-reduction.ll
-Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
-Transforms/LoopVectorize/X86/float-induction-x86.ll
-Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll
-Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll
-Transforms/LoopVectorize/X86/gather_scatter.ll
-Transforms/LoopVectorize/X86/idiv-by-const.ll
-Transforms/LoopVectorize/X86/induction-costs.ll
-Transforms/LoopVectorize/X86/interleaved-accesses-use-after-free.ll
-Transforms/LoopVectorize/X86/intrinsiccost.ll
-Transforms/LoopVectorize/X86/invariant-load-gather.ll
-Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
-Transforms/LoopVectorize/X86/iv-live-outs.ll
-Transforms/LoopVectorize/X86/masked-store-cost.ll
-Transforms/LoopVectorize/X86/masked_load_store.ll
-Transforms/LoopVectorize/X86/multi-exit-cost.ll
-Transforms/LoopVectorize/X86/optsize.ll
-Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
-Transforms/LoopVectorize/X86/pr23997.ll
-Transforms/LoopVectorize/X86/pr47437.ll
-Transforms/LoopVectorize/X86/pr54634.ll
-Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
-Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
-Transforms/LoopVectorize/X86/propagate-metadata.ll
-Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
-Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
-Transforms/LoopVectorize/X86/scatter_crash.ll
-Transforms/LoopVectorize/X86/small-size.ll
-Transforms/LoopVectorize/X86/strided_load_cost.ll
-Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
-Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-live-outs.ll
-Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
-Transforms/LoopVectorize/X86/uniformshift.ll
-Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
-Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
-Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
-Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
-Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
-Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
-Transforms/LoopVectorize/X86/x86-predication.ll
-Transforms/LoopVectorize/alias-mask-needs-freeze.ll
-Transforms/LoopVectorize/alias-mask.ll
-Transforms/LoopVectorize/as_cast.ll
-Transforms/LoopVectorize/bounded-load-multi-exit.ll
-Transforms/LoopVectorize/cast-costs.ll
-Transforms/LoopVectorize/cast-induction.ll
-Transforms/LoopVectorize/conditional-scalar-assignment-interleave-only.ll
-Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
-Transforms/LoopVectorize/constant-fold-commutative-and.ll
-Transforms/LoopVectorize/constantfolder.ll
-Transforms/LoopVectorize/cse-casts.ll
-Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll
-Transforms/LoopVectorize/early-exit-calls.ll
-Transforms/LoopVectorize/early-exit-load-live-out.ll
-Transforms/LoopVectorize/early-exit-minmax-trip-count.ll
-Transforms/LoopVectorize/early-exit-unary-ops.ll
-Transforms/LoopVectorize/early_exit_legality.ll
-Transforms/LoopVectorize/early_exit_store_legality.ll
-Transforms/LoopVectorize/early_exit_with_outer_loop.ll
-Transforms/LoopVectorize/epilog-iv-select-cmp.ll
-Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
-Transforms/LoopVectorize/epilog-vectorization-dead-epilogue.ll
-Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
-Transforms/LoopVectorize/epilog-vectorization-fmaxnum-reductions.ll
-Transforms/LoopVectorize/epilog-vectorization-reductions.ll
-Transforms/LoopVectorize/epilog-vectorization-scev-expansion.ll
-Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
-Transforms/LoopVectorize/fcmp-uno-fold-interleave.ll
-Transforms/LoopVectorize/find-last-iv-interleave.ll
-Transforms/LoopVectorize/find-last-iv-sinkable-expr-epilogue.ll
-Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
-Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
-Transforms/LoopVectorize/find-last-iv-sinkable-load.ll
-Transforms/LoopVectorize/find-last.ll
-Transforms/LoopVectorize/first-order-recurrence-complex.ll
-Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
-Transforms/LoopVectorize/first-order-recurrence.ll
-Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
-Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
-Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
-Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
-Transforms/LoopVectorize/if-conversion.ll
-Transforms/LoopVectorize/if-pred-non-void.ll
-Transforms/LoopVectorize/if-pred-stores.ll
-Transforms/LoopVectorize/induction.ll
-Transforms/LoopVectorize/interleave-and-scalarize-only.ll
-Transforms/LoopVectorize/interleaved-accesses-2.ll
-Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
-Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
-Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll
-Transforms/LoopVectorize/interleaved-accesses.ll
-Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
-Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
-Transforms/LoopVectorize/iv-select-cmp-nested-loop.ll
-Transforms/LoopVectorize/iv-select-cmp-no-wrap.ll
-Transforms/LoopVectorize/iv-select-cmp-non-const-iv-start.ll
-Transforms/LoopVectorize/iv-select-cmp-trunc.ll
-Transforms/LoopVectorize/iv-select-cmp.ll
-Transforms/LoopVectorize/lcssa-crashes.ll
-Transforms/LoopVectorize/loop-form.ll
-Transforms/LoopVectorize/loop-with-constant-exit-condition.ll
-Transforms/LoopVectorize/memdep-fold-tail.ll
-Transforms/LoopVectorize/minmax_reduction.ll
-Transforms/LoopVectorize/multi_early_exit.ll
-Transforms/LoopVectorize/multiple-argmin-argmax.ll
-Transforms/LoopVectorize/multiple-early-exits.ll
-Transforms/LoopVectorize/multiple-exits-versioning.ll
-Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
-Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
-Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
-Transforms/LoopVectorize/optimal-epilog-vectorization.ll
-Transforms/LoopVectorize/optsize.ll
-Transforms/LoopVectorize/phi-with-fastflags.ll
-Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
-Transforms/LoopVectorize/pr32859.ll
-Transforms/LoopVectorize/pr33706.ll
-Transforms/LoopVectorize/pr34681.ll
-Transforms/LoopVectorize/pr43166-fold-tail-by-masking.ll
-Transforms/LoopVectorize/pr44488-predication.ll
-Transforms/LoopVectorize/pr45525.ll
-Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
-Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
-Transforms/LoopVectorize/pr55100-expand-scev-predicate-used.ll
-Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
-Transforms/LoopVectorize/predicated-early-exits-interleave.ll
-Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
-Transforms/LoopVectorize/predicated-multiple-exits.ll
-Transforms/LoopVectorize/predicated-single-exit.ll
-Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
-Transforms/LoopVectorize/predicator.ll
-Transforms/LoopVectorize/reduction-inloop-uf4.ll
-Transforms/LoopVectorize/reduction-minmax-users-and-predicated.ll
-Transforms/LoopVectorize/reduction-small-size.ll
-Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
-Transforms/LoopVectorize/scalar-select.ll
-Transforms/LoopVectorize/scalarize-masked-call.ll
-Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
-Transforms/LoopVectorize/scev-predicate-reasoning.ll
-Transforms/LoopVectorize/select-cmp-blend-chain.ll
-Transforms/LoopVectorize/select-cmp-predicated.ll
-Transforms/LoopVectorize/select-cmp.ll
-Transforms/LoopVectorize/select-index-interleaving.ll
-Transforms/LoopVectorize/select-min-max-last-index-epilogue.ll
-Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll
-Transforms/LoopVectorize/select-reduction.ll
-Transforms/LoopVectorize/select-smax-last-index.ll
-Transforms/LoopVectorize/select-smin-first-index.ll
-Transforms/LoopVectorize/select-smin-last-index.ll
-Transforms/LoopVectorize/select-umax-last-index.ll
-Transforms/LoopVectorize/select-umin-first-index.ll
-Transforms/LoopVectorize/select-umin-last-index.ll
-Transforms/LoopVectorize/single-early-exit-cond-poison.ll
-Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
-Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
-Transforms/LoopVectorize/single-early-exit-interleave-only.ll
-Transforms/LoopVectorize/single-early-exit-interleave.ll
-Transforms/LoopVectorize/single_early_exit.ll
-Transforms/LoopVectorize/single_early_exit_live_outs.ll
-Transforms/LoopVectorize/sink-to-early-exit.ll
-Transforms/LoopVectorize/smax-idx.ll
-Transforms/LoopVectorize/strict-fadd-interleave-only.ll
-Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
-Transforms/LoopVectorize/tail-folding-counting-down.ll
-Transforms/LoopVectorize/tail-folding-masked-mem-opts.ll
-Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll
-Transforms/LoopVectorize/tail-folding-replicate-region.ll
-Transforms/LoopVectorize/tail-folding-switch.ll
-Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
-Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
-Transforms/LoopVectorize/uniform-blend.ll
-Transforms/LoopVectorize/use-scalar-epilogue-if-tp-fails.ll
-Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll
-Transforms/LoopVectorize/vect.stats.ll
-Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
-Transforms/LoopVectorize/vector-loop-backedge-elimination-predicated-early-exit.ll
-Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
-Transforms/LoopVectorize/version-stride-with-integer-casts.ll
-Transforms/LoopVectorize/vplan-based-stride-mv.ll
Transforms/LowerAtomic/atomic-load.ll
Transforms/LowerAtomic/atomic-swap.ll
Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
>From 3bc39aba4a2c5739bbb43b3a76b4b3da9c0fc965 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 14 Aug 2026 10:34:36 +0100
Subject: [PATCH 4/7] !fixup address comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 99 +++---
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 5 +-
.../branch-weights-match-block-frequencies.ll | 317 ++++++++++++++++++
4 files changed, 377 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 62834bf5ae8d3..b614ef856d114 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6575,7 +6575,7 @@ static bool verifyBlockProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
Tolerance)
continue;
- errs() << "Block probability mismatch for " << VPBB->getName() << ": VPlan "
+ errs() << "Block frequency mismatch for " << VPBB->getName() << ": VPlan "
<< Computed << ", BlockFrequencyInfo " << Expected << "\n";
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 4c9979ce36267..69abc9bef486c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -12,7 +12,9 @@
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/MemoryLocation.h"
@@ -1056,63 +1058,74 @@ SmallVector<VPUser *> vputils::collectUsersRecursively(VPValue *V) {
return Users.takeVector();
}
-/// Returns the probability of the edge from \p Src to \p Dst, taken from the
-/// branch weights recorded on Src's terminator, or unknown if not available.
-/// See llvm::getBranchProbability in llvm/Transforms/Utils/LoopUtils.h for the
-/// IR version.
-static BranchProbability getEdgeProbability(const VPBasicBlock *Src,
- const VPBasicBlock *Dst) {
- // With a single successor the edge is always taken.
- ArrayRef<VPBlockBase *> Successors = Src->getSuccessors();
+/// Returns the probability of reaching each unique successor of \p VPBB, taken
+/// from the branch weights recorded on its terminator, or unknown if not
+/// available. See llvm::getBranchProbability in
+/// llvm/Transforms/Utils/LoopUtils.h for the IR version.
+static SmallVector<std::pair<const VPBasicBlock *, BranchProbability>, 2>
+getSuccessorProbabilities(const VPBasicBlock *VPBB) {
+ ArrayRef<VPBlockBase *> Successors = VPBB->getSuccessors();
+ // With a single successor the edge is always taken and needs no weights.
if (Successors.size() == 1)
- return BranchProbability::getOne();
+ return {{cast<VPBasicBlock>(Successors[0]), BranchProbability::getOne()}};
- auto *Term = dyn_cast_if_present<VPInstruction>(Src->getTerminator());
SmallVector<uint32_t> Weights;
- if (!Term ||
- !extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) ||
- Weights.size() != Successors.size())
- return BranchProbability::getUnknown();
-
- uint64_t Total = sum_of(Weights, uint64_t(0));
+ uint64_t Total = 0;
+ auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
+ if (Term &&
+ extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) &&
+ Weights.size() == Successors.size())
+ Total = sum_of(Weights, uint64_t(0));
+ // Without usable weights all successors have unknown probability. Zero the
+ // weights, so the accumulation below still visits each of them.
if (Total == 0)
- return BranchProbability::getUnknown();
-
- // Sum the weights of all edges from Src to Dst; the same block may be the
- // destination of multiple successors, e.g. for switches.
- uint64_t ToDst = 0;
- for (const auto &[Succ, Weight] : zip(Successors, Weights))
- if (Succ == Dst)
- ToDst += Weight;
- return BranchProbability::getBranchProbability(ToDst, Total);
+ Weights.assign(Successors.size(), 0);
+
+ SmallMapVector<const VPBasicBlock *, uint64_t, 2> WeightPerSuccessor;
+ for (const auto &[Succ, Weight] : zip_equal(Successors, Weights))
+ WeightPerSuccessor[cast<VPBasicBlock>(Succ)] += Weight;
+
+ return map_to_vector<2>(WeightPerSuccessor, [Total](const auto &SuccWeight) {
+ auto [Succ, Weight] = SuccWeight;
+ return std::make_pair(
+ Succ, Total == 0
+ ? BranchProbability::getUnknown()
+ : BranchProbability::getBranchProbability(Weight, Total));
+ });
}
DenseMap<const VPBasicBlock *, BranchProbability>
vputils::computeBlockProbabilities(ArrayRef<VPBasicBlock *> Blocks) {
assert(!Blocks.empty() && "expected at least the header block");
+ // Push each block's probability along its outgoing edges, accumulating it in
+ // the successors. Blocks is in reverse post-order and the blocks form a DAG
+ // (the backedge of a loop region is implicit), so all incoming edges of a
+ // block have contributed by the time it is visited and its probability is
+ // final.
DenseMap<const VPBasicBlock *, BranchProbability> Probabilities;
- // The header (first block) always executes. Any other block executes if any
- // of its incoming edges is taken, so accumulate their probabilities.
+ Probabilities.reserve(Blocks.size());
+ // The header (first block) always executes.
Probabilities[Blocks.front()] = BranchProbability::getOne();
- for (VPBasicBlock *VPBB : Blocks.drop_front()) {
- BranchProbability Prob = BranchProbability::getZero();
- // A predecessor may be listed once per edge to VPBB, e.g. for a switch with
- // multiple cases branching here.
- SmallSetVector<VPBlockBase *, 4> Preds(from_range, VPBB->getPredecessors());
- for (VPBasicBlock *PredVPBB : VPBlockUtils::blocksAs<VPBasicBlock>(Preds)) {
- BranchProbability PredProb = Probabilities.lookup(PredVPBB);
- BranchProbability EdgeProb = getEdgeProbability(PredVPBB, VPBB);
- if (PredProb.isUnknown() || EdgeProb.isUnknown()) {
- Prob = BranchProbability::getUnknown();
- break;
+ for (VPBasicBlock *VPBB : Blocks.drop_front())
+ Probabilities[VPBB] = BranchProbability::getZero();
+
+ for (VPBasicBlock *VPBB : Blocks) {
+ BranchProbability SrcProb = Probabilities.at(VPBB);
+ for (auto [Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
+ BranchProbability &SuccProb = Probabilities.at(Succ);
+ // An unknown edge or predecessor poisons the successor: its probability
+ // is only known if all edges on paths reaching it carry branch weights.
+ if (SrcProb.isUnknown() || EdgeProb.isUnknown() || SuccProb.isUnknown()) {
+ SuccProb = BranchProbability::getUnknown();
+ continue;
}
- BranchProbability Contribution = PredProb * EdgeProb;
- // Force to lowest possible probability if result gets rounded to zero.
- if (Contribution.isZero() && !PredProb.isZero() && !EdgeProb.isZero())
+ BranchProbability Contribution = SrcProb * EdgeProb;
+ // Force to the lowest possible probability if the product gets rounded to
+ // zero, to keep reachable blocks distinguishable from unreachable ones.
+ if (Contribution.isZero() && !SrcProb.isZero() && !EdgeProb.isZero())
Contribution = BranchProbability::getRaw(1);
- Prob += Contribution;
+ SuccProb += Contribution;
}
- Probabilities[VPBB] = Prob;
}
return Probabilities;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 1b3d55e65b4e2..aa486c3831510 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -224,8 +224,9 @@ VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
/// Computes for each block in \p Blocks the probability that it executes,
/// relative to the first block in \p Blocks (the header block), which always
-/// executes. The probability of a block is the accumulated probability of its
-/// incoming edges.
+/// executes. \p Blocks must be in reverse post-order. The probability of a
+/// block is the accumulated probability of its incoming edges, or unknown if
+/// any edge on a path reaching it lacks branch weights.
DenseMap<const VPBasicBlock *, BranchProbability>
computeBlockProbabilities(ArrayRef<VPBasicBlock *> Blocks);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
index 0cd3385a11d1d..012c01f621081 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
@@ -310,8 +310,325 @@ exit:
ret void
}
+define void @switch_common_dest_weight_sum_not_a_power_of_two(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; %if.then is reached from both of the switch's cases, %default via the default
+; edge. The weights do not sum to a power of two, so converting each edge to a
+; probability on its own does not divide evenly.
+;
+; %loop 1000/1000 = 1
+; %if.then 667/1000 = 2/3
+; %default 333/1000 = 1/3
+; %latch 1000/1000 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_common_dest_weight_sum_not_a_power_of_two
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - default: float = 333.33,
+; BFI-NEXT: - if.then: float = 666.67,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_common_dest_weight_sum_not_a_power_of_two'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep.idx>
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN-NEXT: EMIT vp<[[C0:%.+]]> = icmp eq ir<%l>, ir<0>
+; VPLAN-NEXT: EMIT vp<[[C1:%.+]]> = icmp eq ir<%l>, ir<1>
+; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
+; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!prof {1431655765, 715827883})
+; VPLAN-NEXT: Successor(s): default
+; VPLAN-EMPTY:
+; VPLAN-NEXT: default:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!prof {715827883, 1431655765})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i8, ptr %idx, i64 %iv
+ %l = load i8, ptr %gep.idx, align 1
+ switch i8 %l, label %default [
+ i8 0, label %if.then
+ i8 1, label %if.then
+ ], !prof !5
+
+default:
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ store i8 0, ptr %gep.b, align 1
+ br label %latch
+
+if.then:
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ store i8 1, ptr %gep.a, align 1
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @switch_common_dest_almost_always_taken(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; %if.then is reached from both of the switch's cases, each taken with a
+; probability just under 1/2, so together they are taken almost always.
+;
+; %loop 1 = 1
+; %if.then 4294967294/4294967295 ~ 1
+; %default 1/4294967295 ~ 0
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_common_dest_almost_always_taken
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - default: float = 0.00000046566,
+; BFI-NEXT: - if.then: float = 1000.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_common_dest_almost_always_taken'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep.idx>
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN-NEXT: EMIT vp<[[C0:%.+]]> = icmp eq ir<%l>, ir<0>
+; VPLAN-NEXT: EMIT vp<[[C1:%.+]]> = icmp eq ir<%l>, ir<1>
+; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
+; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!prof {2147483647, 1})
+; VPLAN-NEXT: Successor(s): default
+; VPLAN-EMPTY:
+; VPLAN-NEXT: default:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!prof {1, 2147483647})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i8, ptr %idx, i64 %iv
+ %l = load i8, ptr %gep.idx, align 1
+ switch i8 %l, label %default [
+ i8 0, label %if.then
+ i8 1, label %if.then
+ ], !prof !6
+
+default:
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ store i8 0, ptr %gep.b, align 1
+ br label %latch
+
+if.then:
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ store i8 1, ptr %gep.a, align 1
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; %mid is almost never executed, and %if.then is reached from 4 of the switch's
+; cases in %mid. The probability of each single edge to %if.then rounds to zero,
+; so each is bumped to the lowest representable probability.
+;
+; %loop 1 = 1
+; %mid 1/2147483648 ~ 0
+; %if.then 4/5 * 1/2147483648 ~ 0
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_common_dest_almost_never_taken
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - mid: float = 0.00000046566,
+; BFI-NEXT: - if.then: float = 0.00000037253,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_common_dest_almost_never_taken'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep.idx>
+; VPLAN-NEXT: EMIT ir<%c> = icmp sgt ir<%l>, ir<0>
+; VPLAN-NEXT: Successor(s): mid
+; VPLAN-EMPTY:
+; VPLAN-NEXT: mid:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!prof {1, 2147483647})
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN-NEXT: EMIT vp<[[C1:%.+]]> = icmp eq ir<%l>, ir<1>
+; VPLAN-NEXT: EMIT vp<[[C2:%.+]]> = icmp eq ir<%l>, ir<2>
+; VPLAN-NEXT: EMIT vp<[[C3:%.+]]> = icmp eq ir<%l>, ir<3>
+; VPLAN-NEXT: EMIT vp<[[C4:%.+]]> = icmp eq ir<%l>, ir<4>
+; VPLAN-NEXT: EMIT vp<[[OR_0:%.+]]> = or vp<[[C1]]>, vp<[[C2]]>
+; VPLAN-NEXT: EMIT vp<[[OR_1:%.+]]> = or vp<[[OR_0]]>, vp<[[C3]]>
+; VPLAN-NEXT: EMIT vp<[[ANY:%.+]]> = or vp<[[OR_1]]>, vp<[[C4]]>
+; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = logical-and ir<%c>, vp<[[ANY]]>
+; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
+; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!prof {1, 2147483647})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i8, ptr %idx, i64 %iv
+ %l = load i8, ptr %gep.idx, align 1
+ %c = icmp sgt i8 %l, 0
+ br i1 %c, label %mid, label %latch, !prof !7
+
+mid:
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ store i8 0, ptr %gep.b, align 1
+ switch i8 %l, label %latch [
+ i8 1, label %if.then
+ i8 2, label %if.then
+ i8 3, label %if.then
+ i8 4, label %if.then
+ ], !prof !8
+
+if.then:
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ store i8 1, ptr %gep.a, align 1
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; Same as @switch_common_dest_almost_never_taken, but with enough edges to
+; %if.then that bumping each of them separately to the lowest representable
+; probability drifts further from BlockFrequencyInfo than its tolerance allows.
+;
+; %loop 1 = 1
+; %mid 1/2147483648 ~ 0
+; %if.then 8/9 * 1/2147483648 ~ 0
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_common_dest_many_edges_almost_never_taken
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - mid: float = 0.00000046566,
+; BFI-NEXT: - if.then: float = 0.00000041392,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_common_dest_many_edges_almost_never_taken'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep.idx>
+; VPLAN-NEXT: EMIT ir<%c> = icmp sgt ir<%l>, ir<0>
+; VPLAN-NEXT: Successor(s): mid
+; VPLAN-EMPTY:
+; VPLAN-NEXT: mid:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!prof {1, 2147483647})
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN: EMIT vp<[[MASK:%.+]]> = logical-and ir<%c>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
+; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!prof {1, 2147483647})
+; VPLAN-NEXT: Successor(s): latch
+; VPLAN-EMPTY:
+; VPLAN-NEXT: latch:
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i8, ptr %idx, i64 %iv
+ %l = load i8, ptr %gep.idx, align 1
+ %c = icmp sgt i8 %l, 0
+ br i1 %c, label %mid, label %latch, !prof !7
+
+mid:
+ %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+ store i8 0, ptr %gep.b, align 1
+ switch i8 %l, label %latch [
+ i8 1, label %if.then
+ i8 2, label %if.then
+ i8 3, label %if.then
+ i8 4, label %if.then
+ i8 5, label %if.then
+ i8 6, label %if.then
+ i8 7, label %if.then
+ i8 8, label %if.then
+ ], !prof !9
+
+if.then:
+ %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
+ store i8 1, ptr %gep.a, align 1
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
!0 = !{!"branch_weights", i32 1, i32 3}
!1 = !{!"branch_weights", i32 1, i32 2}
!2 = !{!"branch_weights", i32 1, i32 1}
!3 = !{!"branch_weights", i32 1, i32 999}
!4 = !{!"branch_weights", i32 500, i32 125, i32 250, i32 125}
+!5 = !{!"branch_weights", i32 1, i32 1, i32 1}
+!6 = !{!"branch_weights", i32 1, i32 2147483647, i32 2147483647}
+!7 = !{!"branch_weights", i32 1, i32 2147483647}
+!8 = !{!"branch_weights", i32 1, i32 1, i32 1, i32 1, i32 1}
+!9 = !{!"branch_weights", i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1}
>From cafe0271a823929227f444b482824ae7932a751f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 19 Aug 2026 20:00:25 +0100
Subject: [PATCH 5/7] !fixup address comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 45 ++++++++++--------
llvm/lib/Transforms/Vectorize/VPlan.h | 36 ++++++++++++--
.../Transforms/Vectorize/VPlanPredicator.cpp | 41 +++++-----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 44 ++++++++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 32 ++++++-------
.../Transforms/Vectorize/VPlanTransforms.h | 7 +--
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 30 ++++++++++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 47 +++++++------------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 9 +---
...-probabilities-match-block-frequencies.ll} | 42 ++++++++---------
.../VPlan/vplan-print-before-after-all.ll | 2 +-
.../VPlan/vplan-printing-branch-weights.ll | 14 +++---
12 files changed, 202 insertions(+), 147 deletions(-)
rename llvm/test/Transforms/LoopVectorize/VPlan/{branch-weights-match-block-frequencies.ll => execution-probabilities-match-block-frequencies.ll} (94%)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e7b5cb3ab6b83..2d7139de1051a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6518,27 +6518,30 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
static void printOptimizedVPlan(VPlan &) {}
#ifndef NDEBUG
-/// Cross-check the probabilities vputils::computeBlockProbabilities computes
-/// for the blocks of the loop region of \p Plan against the frequencies \p BFI
-/// computed for the corresponding blocks of \p OrigLoop.
+/// Cross-check the execution probabilities computed by
+/// vputils::computeExecutionProbabilities for the blocks of the loop region of
+/// \p Plan against the frequencies \p BFI computed for the corresponding blocks
+/// of \p OrigLoop.
/// FIXME: Temporary verification aid, to be removed.
-static bool verifyBlockProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
- LoopInfo *LI,
- BlockFrequencyInfo &BFI) {
-
- // The verification is limited inner loops where the latch is the only exiting
- // block and there are no extra VPBBs not mapped to IR BBs (when tailfolding).
+static bool verifyExecutionProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
+ LoopInfo *LI,
+ BlockFrequencyInfo &BFI) {
+ // The verification is limited to inner loops where the latch is the only
+ // exiting block and there are no extra VPBBs not mapped to IR BBs (when
+ // tail folding).
if (Plan.isOuterLoop() ||
OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch() ||
Plan.hasTailFolded())
return true;
// Visit the blocks of the loop region in the same order as
- // introduceMasksAndLinearize does.
+ // introduceMasksAndLinearize does. Both traversals are reverse post-orders of
+ // the same CFG, so they visit corresponding blocks at the same index.
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion()->getEntryBasicBlock());
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- assert(Blocks.size() == OrigLoop->getNumBlocks());
+ assert(Blocks.size() == OrigLoop->getNumBlocks() &&
+ "loop region and original loop must have the same blocks");
LoopBlocksRPO OrigRPO(OrigLoop);
OrigRPO.perform(LI);
@@ -6550,13 +6553,13 @@ static bool verifyBlockProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
// BFI's fixed-point mass propagation rounds per edge, losing up to 1 ULP per
// block on the path from the header.
uint64_t Tolerance =
- Blocks.size() + BranchProbability::getDenominator() / HeaderFreq;
+ Blocks.size() + VPExecutionProbability::getDenominator() / HeaderFreq;
- DenseMap<const VPBasicBlock *, BranchProbability> Probabilities =
- vputils::computeBlockProbabilities(Blocks);
+ DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities =
+ vputils::computeExecutionProbabilities(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
- BranchProbability Computed = Probabilities.lookup(VPBB);
+ VPExecutionProbability Computed = Probabilities.lookup(VPBB);
// Currently VPlan-based probabilities are only computed when all blocks
// have branch-weights.
if (Computed.isUnknown())
@@ -6565,8 +6568,9 @@ static bool verifyBlockProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
// Clamp the frequency to the header's; it may exceed it slightly due to
// BFI's rounding.
uint64_t Freq = BFI.getBlockFreq(BB).getFrequency();
- BranchProbability Expected = BranchProbability::getBranchProbability(
- std::min(Freq, HeaderFreq), HeaderFreq);
+ VPExecutionProbability Expected =
+ VPExecutionProbability::getBranchProbability(std::min(Freq, HeaderFreq),
+ HeaderFreq);
if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
Tolerance)
continue;
@@ -6665,8 +6669,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
if (CM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
- assert(verifyBlockProbabilitiesMatchBFI(*VPlan0, OrigLoop, LI, CM.getBFI()) &&
- "block probabilities do not match the original loop's frequencies");
+ assert(verifyExecutionProbabilitiesMatchBFI(*VPlan0, OrigLoop, LI,
+ CM.getBFI()) &&
+ "execution probabilities do not match the loop's frequencies");
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
@@ -6883,7 +6888,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
- RUN_VPLAN_PASS(VPlanTransforms::dropBranchWeightsFromUnguardedRecipes, *Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::dropUnguardedExecutionProbabilities, *Plan);
RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
Range);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 897a08862bd3c..544838343d613 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -38,6 +38,7 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/FMF.h"
#include "llvm/IR/Operator.h"
+#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/InstructionCost.h"
#include <cassert>
@@ -73,6 +74,10 @@ struct VPCostContext;
using VPlanPtr = std::unique_ptr<VPlan>;
+/// The probability with which a block or recipe executes, relative to the entry
+/// of the loop region, which always executes.
+using VPExecutionProbability = BranchProbability;
+
/// \enum UncountableExitStyle
/// Different methods of handling early exits.
///
@@ -1178,6 +1183,19 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
class LLVM_ABI_FOR_TEST VPIRMetadata {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ /// Name of the VPlan-internal metadata kind recording the probability with
+ /// which a recipe executes; see setExecutionProbability.
+ static constexpr StringLiteral ExecutionProbabilityMDName =
+ "vplan.execution.probability";
+
+ /// Returns the ID of the metadata kind named \p Kind. Any attached node
+ /// provides the context the IDs are assigned in, as all nodes belong to the
+ /// context of the VPlan's function.
+ unsigned getMDKindID(StringRef Kind) const {
+ assert(!Metadata.empty() && "no node to take the context from");
+ return Metadata.front().second->getContext().getMDKindID(Kind);
+ }
+
public:
VPIRMetadata() = default;
@@ -1185,6 +1203,8 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
/// \p I.
VPIRMetadata(Instruction &I) {
getMetadataToPropagate(&I, Metadata);
+ // Retain the branch weights of terminators. They are used to compute the
+ // probabilities with which the blocks of the original loop execute.
if (I.isTerminator())
if (MDNode *BW = I.getMetadata(LLVMContext::MD_prof))
Metadata.emplace_back(LLVMContext::MD_prof, BW);
@@ -1215,11 +1235,6 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
/// nodes that are common to both.
void intersect(const VPIRMetadata &MD);
- /// Remove metadata of kind \p Kind, if present.
- void eraseMetadata(unsigned Kind) {
- erase_if(Metadata, [Kind](const auto &P) { return P.first == Kind; });
- }
-
/// Get metadata of kind \p Kind. Returns nullptr if not found.
MDNode *getMetadata(unsigned Kind) const {
auto It =
@@ -1227,6 +1242,17 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
return It != Metadata.end() ? It->second : nullptr;
}
+ /// Record that the recipe executes with probability \p Prob, relative to the
+ /// entry of the loop region.
+ void setExecutionProbability(VPExecutionProbability Prob, LLVMContext &Ctx);
+
+ /// Returns the probability recorded by setExecutionProbability, or an unknown
+ /// probability if none has been recorded.
+ VPExecutionProbability getExecutionProbability() const;
+
+ /// Drop the probability recorded by setExecutionProbability, if any.
+ void clearExecutionProbability();
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print metadata with node IDs.
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index e69447bc87d66..c34f59431b14e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -19,8 +19,6 @@
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
-#include "llvm/IR/MDBuilder.h"
-#include <numeric>
using namespace llvm;
using namespace VPlanPatternMatch;
@@ -114,23 +112,6 @@ class VPPredicator {
};
} // namespace
-/// Returns the branch weights describing how often a block executing with
-/// probability \p Prob is entered, or nullptr if \p Prob carries no useful
-/// information, i.e. the block is never or always executed, or its probability
-/// is unknown.
-static MDNode *getWeightsForProbability(BranchProbability Prob,
- LLVMContext &Ctx) {
- if (Prob.isUnknown() || Prob.isZero() || Prob.isOne())
- return nullptr;
-
- // Use the numerators of Prob and its complement as weights and reduce them
- // via gcd to keep them small.
- uint32_t Taken = Prob.getNumerator();
- uint32_t NotTaken = Prob.getCompl().getNumerator();
- uint32_t GCD = std::gcd(Taken, NotTaken);
- return MDBuilder(Ctx).createBranchWeights(Taken / GCD, NotTaken / GCD);
-}
-
VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src,
const VPBasicBlock *Dst) {
assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge");
@@ -425,8 +406,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Header);
// Non-outer regions with VPBBs only are supported at the moment.
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- DenseMap<const VPBasicBlock *, BranchProbability> Probabilities =
- vputils::computeBlockProbabilities(Blocks);
+ DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities =
+ vputils::computeExecutionProbabilities(Blocks);
VPPredicator Predicator(Plan);
for (VPBasicBlock *VPBB : Blocks) {
@@ -440,16 +421,16 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
if (!BlockMask)
continue;
- // Mask all VPInstructions in the block and propagate execution
- // probabilities to them.
- MDNode *Weights =
- getWeightsForProbability(Probabilities.lookup(VPBB), Plan.getContext());
+ // Mask all VPInstructions in the block and record the probability with
+ // which the masked recipes execute.
+ VPExecutionProbability Prob = Probabilities.lookup(VPBB);
for (VPRecipeBase &R : *VPBB) {
- if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
- VPI->addMask(BlockMask);
- if (Weights && VPI->isMasked())
- VPI->setMetadata(LLVMContext::MD_prof, Weights);
- }
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (!VPI)
+ continue;
+ VPI->addMask(BlockMask);
+ if (VPI->isMasked())
+ VPI->setExecutionProbability(Prob, Plan.getContext());
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a43b8cedc5e13..33d459c5fb9ef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -35,6 +35,7 @@
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -2128,10 +2129,42 @@ void VPIRPhi::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
void VPIRMetadata::applyMetadata(Instruction &I) const {
+ assert(getExecutionProbability().isUnknown() &&
+ "VPlan-internal metadata must not be propagated to IR");
for (const auto &[Kind, Node] : Metadata)
I.setMetadata(Kind, Node);
}
+void VPIRMetadata::setExecutionProbability(VPExecutionProbability Prob,
+ LLVMContext &Ctx) {
+ // A recipe that never or always executes needs no annotation.
+ if (Prob.isUnknown() || Prob.isZero() || Prob.isOne())
+ return;
+ // The numerator is relative to BranchProbability's fixed denominator and is
+ // less than it, as probability one has been excluded above.
+ Constant *Numerator =
+ ConstantInt::get(Type::getInt32Ty(Ctx), Prob.getNumerator());
+ setMetadata(Ctx.getMDKindID(ExecutionProbabilityMDName),
+ MDNode::get(Ctx, {ConstantAsMetadata::get(Numerator)}));
+}
+
+VPExecutionProbability VPIRMetadata::getExecutionProbability() const {
+ if (Metadata.empty())
+ return VPExecutionProbability::getUnknown();
+ MDNode *Node = getMetadata(getMDKindID(ExecutionProbabilityMDName));
+ if (!Node)
+ return VPExecutionProbability::getUnknown();
+ return VPExecutionProbability::getRaw(
+ mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue());
+}
+
+void VPIRMetadata::clearExecutionProbability() {
+ if (Metadata.empty())
+ return;
+ unsigned ID = getMDKindID(ExecutionProbabilityMDName);
+ erase_if(Metadata, [ID](const auto &P) { return P.first == ID; });
+}
+
void VPIRMetadata::intersect(const VPIRMetadata &Other) {
SmallVector<std::pair<unsigned, MDNode *>> MetadataIntersection;
for (const auto &[KindA, MDA] : Metadata) {
@@ -2158,14 +2191,21 @@ void VPIRMetadata::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
assert(Kind < MDNames.size() && !MDNames[Kind].empty() &&
"Unexpected unnamed metadata kind");
O << "!" << MDNames[Kind] << " ";
+ // Print the values of branch weights, which are more informative than the
+ // ID of the metadata node holding them.
SmallVector<uint32_t> Weights;
if (Kind == LLVMContext::MD_prof && extractBranchWeights(Node, Weights)) {
O << "{";
interleaveComma(Weights, O);
O << "}";
- return;
+ } else if (MDNames[Kind] == ExecutionProbabilityMDName) {
+ // Print execution probabilities as percentages, which is precise enough
+ // to relate them to the block frequencies of the original loop.
+ O << format("%.2f%%", 100.0 * getExecutionProbability().getNumerator() /
+ VPExecutionProbability::getDenominator());
+ } else {
+ Node->printAsOperand(O, M);
}
- Node->printAsOperand(O, M);
});
O << ")";
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 274025c8b9af2..d4ce0dd4cc3bb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -485,12 +485,11 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
// The merged region is entered whenever either of the original regions was,
// so use the higher, i.e. more conservative, of their entry probabilities.
- BranchProbability Prob1 = vputils::getRegionEntryProbability(Region1);
- BranchProbability Prob2 = vputils::getRegionEntryProbability(Region2);
- if (!Prob1.isUnknown() && !Prob2.isUnknown() && Prob2 < Prob1)
- Region2->getEntryBranchOnMask()->setMetadata(
- LLVMContext::MD_prof,
- Region1->getEntryBranchOnMask()->getMetadata(LLVMContext::MD_prof));
+ VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
+ VPExecutionProbability Prob1 =
+ Region1->getEntryBranchOnMask()->getExecutionProbability();
+ if (!Prob1.isUnknown() && Guard2->getExecutionProbability() < Prob1)
+ Guard2->setExecutionProbability(Prob1, Plan.getContext());
// Note: No fusion-preventing memory dependencies are expected in either
// region. Such dependencies should be rejected during earlier dependence
@@ -559,12 +558,11 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
PredRecipe->getDebugLoc());
- // Move the predicated recipes's branch weights onto the guarding
- // branch-on-mask.
- if (MDNode *BW = RecipeWithoutMask->getMetadata(LLVMContext::MD_prof)) {
- BOMRecipe->setMetadata(LLVMContext::MD_prof, BW);
- RecipeWithoutMask->eraseMetadata(LLVMContext::MD_prof);
- }
+ // The predicated recipe executes exactly when the guarding branch-on-mask is
+ // taken, so move its execution probability there.
+ BOMRecipe->setExecutionProbability(
+ RecipeWithoutMask->getExecutionProbability(), Plan.getContext());
+ RecipeWithoutMask->clearExecutionProbability();
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
auto *Exiting = Plan.createVPBasicBlock(Twine(RegionName) + ".continue");
@@ -3722,9 +3720,9 @@ static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
VPIRMetadata CommonMetadata = *Recipes.front();
for (VPReplicateRecipe *Recipe : drop_begin(Recipes))
CommonMetadata.intersect(*Recipe);
- // The recipe the common metadata is used for is not predicated, so drop
- // !prof.
- CommonMetadata.eraseMetadata(LLVMContext::MD_prof);
+ // The recipe the common metadata is used for is not predicated, so it does
+ // not execute with the probability shared by the recipes in the group.
+ CommonMetadata.clearExecutionProbability();
return CommonMetadata;
}
@@ -5583,7 +5581,7 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
}
}
-void VPlanTransforms::dropBranchWeightsFromUnguardedRecipes(VPlan &Plan) {
+void VPlanTransforms::dropUnguardedExecutionProbabilities(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : *VPBB) {
@@ -5591,7 +5589,7 @@ void VPlanTransforms::dropBranchWeightsFromUnguardedRecipes(VPlan &Plan) {
if (RepR && RepR->isPredicated())
continue;
if (auto *MD = dyn_cast<VPIRMetadata>(&R))
- MD->eraseMetadata(LLVMContext::MD_prof);
+ MD->clearExecutionProbability();
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 8beaf0f392921..497168277604a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -617,9 +617,10 @@ struct VPlanTransforms {
/// enabled by prior widening of consecutive memory operations for now.
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
- /// Drop the branch weights from all recipes that cannot preserve them.
- /// Currently that is all recipes, except VPReplicateRecipes.
- static void dropBranchWeightsFromUnguardedRecipes(VPlan &Plan);
+ /// Drop the execution probabilities recorded on all recipes that will not be
+ /// guarded by a branch. Currently those are all recipes except predicated
+ /// VPReplicateRecipes.
+ static void dropUnguardedExecutionProbabilities(VPlan &Plan);
/// Convert call VPInstructions in \p Plan into widened call, vector
/// intrinsic or replicate recipes based on a cost comparison via \p CostCtx.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 764c3059bc2e9..3e4ff6d1f9879 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -25,6 +25,8 @@
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/MDBuilder.h"
+#include <numeric>
using namespace llvm;
using namespace llvm::VPlanPatternMatch;
@@ -674,6 +676,22 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
return New;
}
+/// Converts the probability \p Prob with which a block is entered to the branch
+/// weights of the conditional branch guarding it, or returns nullptr if \p Prob
+/// is unknown.
+static MDNode *convertProbabilityToBranchWeights(VPExecutionProbability Prob,
+ LLVMContext &Ctx) {
+ if (Prob.isUnknown())
+ return nullptr;
+
+ // Use the numerators of Prob and its complement as weights and reduce them
+ // via gcd to keep them small.
+ uint32_t Taken = Prob.getNumerator();
+ uint32_t NotTaken = Prob.getCompl().getNumerator();
+ uint32_t GCD = std::gcd(Taken, NotTaken);
+ return MDBuilder(Ctx).createBranchWeights(Taken / GCD, NotTaken / GCD);
+}
+
/// Convert recipes in region blocks to operate on a single lane 0.
/// VPReplicateRecipes are converted to single-scalar ones, branch-on-mask is
/// converted into BranchOnCond, PredInstPhi recipes are replaced by scalar phi
@@ -723,11 +741,13 @@ static void convertRecipesInRegionBlocksToSingleScalar(VPlan &Plan, Type *IdxTy,
RepR->replaceAllUsesWith(NewR);
RepR->eraseFromParent();
} else if (auto *BranchOnMask = dyn_cast<VPBranchOnMaskRecipe>(&OldR)) {
- // Carry the branch weights describing how often the predicated block
- // is entered onto the generated conditional branch.
- Builder.createNaryOp(VPInstruction::BranchOnCond,
- {BranchOnMask->getOperand(0)},
- /*Inst=*/nullptr, {}, *BranchOnMask, OldDL);
+ // Turn the probability with which the predicated block is entered into
+ // branch weights on the generated conditional branch.
+ auto *BOC = Builder.createNaryOp(VPInstruction::BranchOnCond,
+ {BranchOnMask->getOperand(0)}, OldDL);
+ if (MDNode *Weights = convertProbabilityToBranchWeights(
+ BranchOnMask->getExecutionProbability(), Plan.getContext()))
+ BOC->setMetadata(LLVMContext::MD_prof, Weights);
BranchOnMask->eraseFromParent();
} else if (auto *PredPhi = dyn_cast<VPPredInstPHIRecipe>(&OldR)) {
VPValue *PredOp = PredPhi->getOperand(0);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 69abc9bef486c..db2cd813d4b28 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1069,17 +1069,16 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
if (Successors.size() == 1)
return {{cast<VPBasicBlock>(Successors[0]), BranchProbability::getOne()}};
+ // Take the branch weights off the terminator. Without usable weights all
+ // successors have unknown probability; zero the weights, so the accumulation
+ // below still visits each of them.
SmallVector<uint32_t> Weights;
- uint64_t Total = 0;
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
- if (Term &&
- extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) &&
- Weights.size() == Successors.size())
- Total = sum_of(Weights, uint64_t(0));
- // Without usable weights all successors have unknown probability. Zero the
- // weights, so the accumulation below still visits each of them.
- if (Total == 0)
+ if (!Term ||
+ !extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) ||
+ Weights.size() != Successors.size())
Weights.assign(Successors.size(), 0);
+ uint64_t Total = sum_of(Weights, uint64_t(0));
SmallMapVector<const VPBasicBlock *, uint64_t, 2> WeightPerSuccessor;
for (const auto &[Succ, Weight] : zip_equal(Successors, Weights))
@@ -1094,54 +1093,42 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
});
}
-DenseMap<const VPBasicBlock *, BranchProbability>
-vputils::computeBlockProbabilities(ArrayRef<VPBasicBlock *> Blocks) {
+DenseMap<const VPBasicBlock *, VPExecutionProbability>
+vputils::computeExecutionProbabilities(ArrayRef<VPBasicBlock *> Blocks) {
assert(!Blocks.empty() && "expected at least the header block");
// Push each block's probability along its outgoing edges, accumulating it in
// the successors. Blocks is in reverse post-order and the blocks form a DAG
// (the backedge of a loop region is implicit), so all incoming edges of a
// block have contributed by the time it is visited and its probability is
// final.
- DenseMap<const VPBasicBlock *, BranchProbability> Probabilities;
+ DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities;
Probabilities.reserve(Blocks.size());
// The header (first block) always executes.
- Probabilities[Blocks.front()] = BranchProbability::getOne();
+ Probabilities[Blocks.front()] = VPExecutionProbability::getOne();
for (VPBasicBlock *VPBB : Blocks.drop_front())
- Probabilities[VPBB] = BranchProbability::getZero();
+ Probabilities[VPBB] = VPExecutionProbability::getZero();
for (VPBasicBlock *VPBB : Blocks) {
- BranchProbability SrcProb = Probabilities.at(VPBB);
+ VPExecutionProbability SrcProb = Probabilities.at(VPBB);
for (auto [Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- BranchProbability &SuccProb = Probabilities.at(Succ);
+ VPExecutionProbability &SuccProb = Probabilities.at(Succ);
// An unknown edge or predecessor poisons the successor: its probability
// is only known if all edges on paths reaching it carry branch weights.
if (SrcProb.isUnknown() || EdgeProb.isUnknown() || SuccProb.isUnknown()) {
- SuccProb = BranchProbability::getUnknown();
+ SuccProb = VPExecutionProbability::getUnknown();
continue;
}
- BranchProbability Contribution = SrcProb * EdgeProb;
+ VPExecutionProbability Contribution = SrcProb * EdgeProb;
// Force to the lowest possible probability if the product gets rounded to
// zero, to keep reachable blocks distinguishable from unreachable ones.
if (Contribution.isZero() && !SrcProb.isZero() && !EdgeProb.isZero())
- Contribution = BranchProbability::getRaw(1);
+ Contribution = VPExecutionProbability::getRaw(1);
SuccProb += Contribution;
}
}
return Probabilities;
}
-BranchProbability
-vputils::getRegionEntryProbability(const VPRegionBlock *Region) {
- const VPBranchOnMaskRecipe *Guard = Region->getEntryBranchOnMask();
- SmallVector<uint32_t, 2> Weights;
- if (!extractBranchWeights(Guard->getMetadata(LLVMContext::MD_prof), Weights))
- return BranchProbability::getUnknown();
- // The weights are {Taken, NotTaken}: the region is entered if the branch on
- // the guarding mask is taken.
- return BranchProbability::getBranchProbability(Weights[0],
- sum_of(Weights, uint64_t(0)));
-}
-
VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
ArrayRef<VPValue *> Operands,
const DataLayout &DL) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index aa486c3831510..ddaa7c076d06d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -227,13 +227,8 @@ VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
/// executes. \p Blocks must be in reverse post-order. The probability of a
/// block is the accumulated probability of its incoming edges, or unknown if
/// any edge on a path reaching it lacks branch weights.
-DenseMap<const VPBasicBlock *, BranchProbability>
-computeBlockProbabilities(ArrayRef<VPBasicBlock *> Blocks);
-
-/// Returns the probability of entering replicate region \p Region, taken from
-/// the branch weights recorded on its guarding branch-on-mask, or unknown if it
-/// carries none.
-BranchProbability getRegionEntryProbability(const VPRegionBlock *Region);
+DenseMap<const VPBasicBlock *, VPExecutionProbability>
+computeExecutionProbabilities(ArrayRef<VPBasicBlock *> Blocks);
namespace detail {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
similarity index 94%
rename from llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
rename to llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
index 012c01f621081..004b8cba88936 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/branch-weights-match-block-frequencies.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
@@ -4,8 +4,8 @@
; RUN: -vplan-print-after=introduceMasksAndLinearize -disable-output %s 2>&1 \
; RUN: | FileCheck --check-prefix=VPLAN %s
-; Check that the branch weights VPlan puts on the masked recipes of a block
-; describe the same execution probability BlockFrequencyInfo computes for the
+; Check that the execution probabilities VPlan records on the masked recipes of
+; a block match the block frequencies BlockFrequencyInfo computes for the
; corresponding block of the original scalar loop.
define void @single_pred(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
@@ -34,7 +34,7 @@ define void @single_pred(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25.00%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -97,20 +97,20 @@ define void @two_preds(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noali
; VPLAN-NEXT: else:
; VPLAN-NEXT: EMIT vp<[[NOT_C0:%.+]]> = not ir<%c.0>
; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!prof {3, 1})
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!prof {3, 1})
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!vplan.execution.probability 75.00%)
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!vplan.execution.probability 75.00%)
; VPLAN-NEXT: Successor(s): then
; VPLAN-EMPTY:
; VPLAN-NEXT: then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25.00%)
; VPLAN-NEXT: Successor(s): merge
; VPLAN-EMPTY:
; VPLAN-NEXT: merge:
; VPLAN-NEXT: EMIT vp<[[AND:%.+]]> = logical-and vp<[[NOT_C0]]>, ir<%c.1>
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = or vp<[[AND]]>, ir<%c.0>
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!prof {1, 1})
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!vplan.execution.probability 50.00%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -178,14 +178,14 @@ define void @nested_ifs(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN-EMPTY:
; VPLAN-NEXT: if.0:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!prof {1, 3})
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!prof {1, 3})
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.probability 25.00%)
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!vplan.execution.probability 25.00%)
; VPLAN-NEXT: Successor(s): if.1
; VPLAN-EMPTY:
; VPLAN-NEXT: if.1:
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = logical-and ir<%c.0>, ir<%c.1>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!prof {1, 7})
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 12.50%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -256,17 +256,17 @@ define void @switch_common_dest(ptr noalias %a, ptr noalias %b, ptr noalias %c,
; VPLAN-NEXT: EMIT vp<[[ANY:%.+]]> = or vp<[[C0_OR_C1]]>, vp<[[C2]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[ANY]]>
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!prof {1, 7})
+; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!vplan.execution.probability 12.50%)
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!prof {3, 5})
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 37.50%)
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!prof {1, 1})
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!vplan.execution.probability 50.00%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -341,12 +341,12 @@ define void @switch_common_dest_weight_sum_not_a_power_of_two(ptr noalias %a, pt
; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!prof {1431655765, 715827883})
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 66.67%)
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!prof {715827883, 1431655765})
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.probability 33.33%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -414,12 +414,12 @@ define void @switch_common_dest_almost_always_taken(ptr noalias %a, ptr noalias
; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!prof {2147483647, 1})
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 100.00%)
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!prof {1, 2147483647})
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.probability 0.00%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -485,7 +485,7 @@ define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %
; VPLAN-EMPTY:
; VPLAN-NEXT: mid:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!prof {1, 2147483647})
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 0.00%)
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
@@ -500,7 +500,7 @@ define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %
; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!prof {1, 2147483647})
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 0.00%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -569,7 +569,7 @@ define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, pt
; VPLAN-EMPTY:
; VPLAN-NEXT: mid:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!prof {1, 2147483647})
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 0.00%)
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
@@ -577,7 +577,7 @@ define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, pt
; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!prof {1, 2147483647})
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 0.00%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index a3c6563a03a01..1bd959b67b672 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -30,7 +30,7 @@
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeScalarizationDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeCallWideningDecisions
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dropBranchWeightsFromUnguardedRecipes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dropUnguardedExecutionProbabilities
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeFindIVReductions
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
index 9e92869f94669..af3ff98adf5d7 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
@@ -12,7 +12,9 @@
; RUN: -vplan-print-after=dissolveLoopRegions -disable-output %s 2>&1 \
; RUN: | FileCheck --strict-whitespace --check-prefix=DISSOLVE %s
-; Track the branch weights of a predicated block through VPlan.
+; Track the execution probability of a predicated block through VPlan, from the
+; branch weights of the original loop to the branch weights of the branch
+; guarding the predicated block.
define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; PREDICATE-LABEL: VPlan for loop in 'predicated_block'
@@ -39,11 +41,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; PREDICATE-NEXT: Successor(s): if.then
; PREDICATE-EMPTY:
; PREDICATE-NEXT: if.then:
-; PREDICATE-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp>
+; PREDICATE-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25.00%)
; PREDICATE-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; PREDICATE-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; PREDICATE-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; PREDICATE-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%cmp>
+; PREDICATE-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25.00%)
; PREDICATE-NEXT: Successor(s): latch
; PREDICATE-EMPTY:
; PREDICATE-NEXT: latch:
@@ -82,11 +84,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; CONSTRUCT-NEXT: Successor(s): if.then
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: if.then:
-; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!prof {1, 3})
+; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25.00%)
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; CONSTRUCT-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!prof {1, 3})
+; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25.00%)
; CONSTRUCT-NEXT: Successor(s): latch
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: latch:
@@ -129,7 +131,7 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; REGION-EMPTY:
; REGION-NEXT: <xVFxUF> pred.store: {
; REGION-NEXT: pred.store.entry:
-; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!prof {1, 3})
+; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!vplan.execution.probability 25.00%)
; REGION-NEXT: Successor(s): pred.store.if, pred.store.continue
; REGION-EMPTY:
; REGION-NEXT: pred.store.if:
>From d7605f352394825ddd24db3470ee8f206d0ffae1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 27 Aug 2026 08:50:18 +0100
Subject: [PATCH 6/7] !fixup keep weights when an edge probability quantizes to
0 or 1
getBranchProbability scales down a total wider than 32 bits and truncates
the numerator with it, which can collapse a taken edge to probability zero,
or a partially taken one to probability one. The block's probability then
becomes zero or one, for which no annotation is recorded, so the guard
branch of the replicate region loses its branch weights entirely. Keep both
ends apart, as BlockFrequencyInfo also gives a zero-weight edge a non-zero
share. Adding one to a single weight no longer changes whether weights are
emitted at all.
Also account for per-edge rounding in the cross-check against
BlockFrequencyInfo, which rounds once per edge rather than once per block,
and print probabilities with %.4g, so that the tiny probabilities of deeply
nested blocks stay legible without adding noise to the common ones.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 20 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 10 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 22 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 1 -
...n-probabilities-match-block-frequencies.ll | 194 +++++++++--
.../VPlan/vplan-printing-branch-weights.ll | 10 +-
.../replicate-region-branch-weights.ll | 302 ++++++++++++------
10 files changed, 432 insertions(+), 162 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 60f259fbcb992..1fb84b2158366 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6485,27 +6485,32 @@ static bool verifyExecutionProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
if (HeaderFreq == 0)
return true;
- // BFI's fixed-point mass propagation rounds per edge, losing up to 1 ULP per
- // block on the path from the header.
- uint64_t Tolerance =
- Blocks.size() + VPExecutionProbability::getDenominator() / HeaderFreq;
+ // BFI's fixed-point mass propagation rounds once per edge, losing up to 1 ULP
+ // per edge on the paths reaching a block. Bound that by the total number of
+ // edges in the region, as a block distributing its frequency over many
+ // successors rounds once for each of them.
+ uint64_t Edges = 0;
+ for (const VPBasicBlock *VPBB : Blocks)
+ Edges += VPBB->getNumSuccessors();
+ uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities =
vputils::computeExecutionProbabilities(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
- VPExecutionProbability Computed = Probabilities.lookup(VPBB);
// Currently VPlan-based probabilities are only computed when all blocks
- // have branch-weights.
- if (Computed.isUnknown())
+ // have branch-weights. Compare at BranchProbability's coarser resolution,
+ // which is as precise as BFI's frequencies get.
+ VPExecutionProbability Prob = Probabilities.lookup(VPBB);
+ if (Prob.isUnknown())
continue;
+ BranchProbability Computed = Prob;
// Clamp the frequency to the header's; it may exceed it slightly due to
// BFI's rounding.
uint64_t Freq = BFI.getBlockFreq(BB).getFrequency();
- VPExecutionProbability Expected =
- VPExecutionProbability::getBranchProbability(std::min(Freq, HeaderFreq),
- HeaderFreq);
+ BranchProbability Expected = BranchProbability::getBranchProbability(
+ std::min(Freq, HeaderFreq), HeaderFreq);
if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
Tolerance)
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a4f6faa149c40..f25b152a84da2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2531,9 +2531,7 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
void setStartValue(VPValue *V) { setOperand(0, V); }
/// Returns the incoming value from the loop backedge.
- virtual VPValue *getBackedgeValue() {
- return getOperand(1);
- }
+ virtual VPValue *getBackedgeValue() { return getOperand(1); }
/// Update the incoming value from the loop backedge.
void setBackedgeValue(VPValue *V) { setOperand(1, V); }
@@ -4432,7 +4430,8 @@ struct CastInfo<VPIRMetadata, VPRecipeBase>
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
-/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
+/// output IR instructions. All PHI-like recipes must come before any non-PHI
+/// recipes.
class LLVM_ABI_FOR_TEST VPBasicBlock : public VPBlockBase {
friend class VPlan;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index eb9f4542dbca3..f4d48c6481032 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2135,6 +2135,12 @@ void VPIRMetadata::applyMetadata(Instruction &I) const {
I.setMetadata(Kind, Node);
}
+/// Returns the execution probability recorded in \p Node.
+static VPExecutionProbability getExecutionProbabilityFromMD(const MDNode *Node) {
+ return VPExecutionProbability::getRaw(
+ mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue());
+}
+
void VPIRMetadata::setExecutionProbability(VPExecutionProbability Prob,
LLVMContext &Ctx) {
// A recipe that never or always executes needs no annotation.
@@ -2154,8 +2160,7 @@ VPExecutionProbability VPIRMetadata::getExecutionProbability() const {
MDNode *Node = getMetadata(getMDKindID(ExecutionProbabilityMDName));
if (!Node)
return VPExecutionProbability::getUnknown();
- return VPExecutionProbability::getRaw(
- mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue());
+ return getExecutionProbabilityFromMD(Node);
}
void VPIRMetadata::clearExecutionProbability() {
@@ -2199,10 +2204,13 @@ void VPIRMetadata::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
interleaveComma(Weights, O);
O << "}";
} else if (MDNames[Kind] == ExecutionProbabilityMDName) {
- // Print execution probabilities as percentages, which is precise enough
- // to relate them to the block frequencies of the original loop.
- O << format("%.2f%%", 100.0 * getExecutionProbability().getNumerator() /
- VPExecutionProbability::getDenominator());
+ // Print execution probabilities as percentages, which relates them to
+ // the block frequencies of the original loop. Use %g with a few
+ // significant digits, so that the tiny probabilities of deeply nested
+ // blocks stay legible without adding noise to the common ones.
+ O << format("%.4g%%",
+ 100.0 * getExecutionProbabilityFromMD(Node).getNumerator() /
+ VPExecutionProbability::getDenominator());
} else {
Node->printAsOperand(O, M);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 064ae700aa4df..e0b287aae1448 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -489,7 +489,8 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
VPExecutionProbability Prob1 =
Region1->getEntryBranchOnMask()->getExecutionProbability();
- if (!Prob1.isUnknown() && Guard2->getExecutionProbability() < Prob1)
+ VPExecutionProbability Prob2 = Guard2->getExecutionProbability();
+ if (!Prob1.isUnknown() && !Prob2.isUnknown() && Prob2 < Prob1)
Guard2->setExecutionProbability(Prob1, Plan.getContext());
// Note: No fusion-preventing memory dependencies are expected in either
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 3e4ff6d1f9879..aa50857834cfe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -683,11 +683,13 @@ static MDNode *convertProbabilityToBranchWeights(VPExecutionProbability Prob,
LLVMContext &Ctx) {
if (Prob.isUnknown())
return nullptr;
+ BranchProbability P = Prob;
- // Use the numerators of Prob and its complement as weights and reduce them
- // via gcd to keep them small.
- uint32_t Taken = Prob.getNumerator();
- uint32_t NotTaken = Prob.getCompl().getNumerator();
+ // Use the numerators of P and its complement as weights and reduce them via
+ // gcd to keep them small. Round a probability that underflows P up, to keep a
+ // rarely entered block distinguishable from a never entered one.
+ uint32_t Taken = std::max(P.getNumerator(), 1u);
+ uint32_t NotTaken = BranchProbability::getDenominator() - Taken;
uint32_t GCD = std::gcd(Taken, NotTaken);
return MDBuilder(Ctx).createBranchWeights(Taken / GCD, NotTaken / GCD);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 92abaf441d577..373209697d601 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -13,6 +13,7 @@
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/TypeSwitch.h"
@@ -1139,16 +1140,27 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
Weights.assign(Successors.size(), 0);
uint64_t Total = sum_of(Weights, uint64_t(0));
+ // Sum the weights of parallel edges to the same successor, so that the
+ // division below rounds once per successor rather than once per edge.
SmallMapVector<const VPBasicBlock *, uint64_t, 2> WeightPerSuccessor;
for (const auto &[Succ, Weight] : zip_equal(Successors, Weights))
WeightPerSuccessor[cast<VPBasicBlock>(Succ)] += Weight;
return map_to_vector<2>(WeightPerSuccessor, [Total](const auto &SuccWeight) {
auto [Succ, Weight] = SuccWeight;
- return std::make_pair(
- Succ, Total == 0
- ? BranchProbability::getUnknown()
- : BranchProbability::getBranchProbability(Weight, Total));
+ if (Total == 0)
+ return std::make_pair(Succ, BranchProbability::getUnknown());
+ BranchProbability P =
+ BranchProbability::getBranchProbability(Weight, Total);
+ // A total wider than 32 bits gets scaled down, truncating the numerator.
+ // That can collapse a taken edge to probability zero, or a partially taken
+ // one to probability one; keep both ends apart, as BlockFrequencyInfo also
+ // gives a zero-weight edge a non-zero share.
+ if (P.isZero() && Weight != 0)
+ P = BranchProbability::getRaw(1);
+ else if (P.isOne() && Weight != Total)
+ P = BranchProbability::getRaw(BranchProbability::getDenominator() - 1);
+ return std::make_pair(Succ, P);
});
}
@@ -1169,7 +1181,7 @@ vputils::computeExecutionProbabilities(ArrayRef<VPBasicBlock *> Blocks) {
for (VPBasicBlock *VPBB : Blocks) {
VPExecutionProbability SrcProb = Probabilities.at(VPBB);
- for (auto [Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
+ for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
VPExecutionProbability &SuccProb = Probabilities.at(Succ);
// An unknown edge or predecessor poisons the successor: its probability
// is only known if all edges on paths reaching it carry branch weights.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index c125bd4f2b239..85d286311cab7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -10,7 +10,6 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANUTILS_H
#include "VPlan.h"
-#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
index 004b8cba88936..1a6290cba5f19 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
@@ -34,7 +34,7 @@ define void @single_pred(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25.00%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -97,20 +97,20 @@ define void @two_preds(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noali
; VPLAN-NEXT: else:
; VPLAN-NEXT: EMIT vp<[[NOT_C0:%.+]]> = not ir<%c.0>
; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!vplan.execution.probability 75.00%)
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!vplan.execution.probability 75.00%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!vplan.execution.probability 75%)
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!vplan.execution.probability 75%)
; VPLAN-NEXT: Successor(s): then
; VPLAN-EMPTY:
; VPLAN-NEXT: then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25.00%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25%)
; VPLAN-NEXT: Successor(s): merge
; VPLAN-EMPTY:
; VPLAN-NEXT: merge:
; VPLAN-NEXT: EMIT vp<[[AND:%.+]]> = logical-and vp<[[NOT_C0]]>, ir<%c.1>
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = or vp<[[AND]]>, ir<%c.0>
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!vplan.execution.probability 50.00%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!vplan.execution.probability 50%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -178,14 +178,14 @@ define void @nested_ifs(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN-EMPTY:
; VPLAN-NEXT: if.0:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.probability 25.00%)
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!vplan.execution.probability 25.00%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.probability 25%)
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!vplan.execution.probability 25%)
; VPLAN-NEXT: Successor(s): if.1
; VPLAN-EMPTY:
; VPLAN-NEXT: if.1:
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = logical-and ir<%c.0>, ir<%c.1>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 12.50%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 12.5%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -256,17 +256,17 @@ define void @switch_common_dest(ptr noalias %a, ptr noalias %b, ptr noalias %c,
; VPLAN-NEXT: EMIT vp<[[ANY:%.+]]> = or vp<[[C0_OR_C1]]>, vp<[[C2]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[ANY]]>
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!vplan.execution.probability 12.50%)
+; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!vplan.execution.probability 12.5%)
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 37.50%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 37.5%)
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!vplan.execution.probability 50.00%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!vplan.execution.probability 50%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -414,12 +414,12 @@ define void @switch_common_dest_almost_always_taken(ptr noalias %a, ptr noalias
; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 100.00%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 100%)
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.probability 0.00%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.probability 4.657e-08%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -459,8 +459,9 @@ exit:
define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; %mid is almost never executed, and %if.then is reached from 4 of the switch's
-; cases in %mid. The probability of each single edge to %if.then rounds to zero,
-; so each is bumped to the lowest representable probability.
+; cases in %mid. Both probabilities are far below BranchProbability's 2^-31
+; resolution, so they must be represented as block frequencies to stay
+; distinguishable from zero.
;
; %loop 1 = 1
; %mid 1/2147483648 ~ 0
@@ -485,7 +486,7 @@ define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %
; VPLAN-EMPTY:
; VPLAN-NEXT: mid:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 0.00%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 4.657e-08%)
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
@@ -500,7 +501,7 @@ define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %
; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 0.00%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 4.657e-08%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -542,9 +543,9 @@ exit:
}
define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
-; Same as @switch_common_dest_almost_never_taken, but with enough edges to
-; %if.then that bumping each of them separately to the lowest representable
-; probability drifts further from BlockFrequencyInfo than its tolerance allows.
+; Same as @switch_common_dest_almost_never_taken, but with more parallel edges
+; to %if.then. Their weights must be summed per successor before the edge
+; probability is computed, so that it is rounded once rather than once per edge.
;
; %loop 1 = 1
; %mid 1/2147483648 ~ 0
@@ -569,7 +570,7 @@ define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, pt
; VPLAN-EMPTY:
; VPLAN-NEXT: mid:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 0.00%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 4.657e-08%)
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
@@ -577,7 +578,7 @@ define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, pt
; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 0.00%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 4.657e-08%)
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -622,6 +623,155 @@ exit:
ret void
}
+define void @switch_common_dest_weight_sum_exceeds_32_bits(ptr noalias %a, ptr noalias %idx) {
+; The weights of the five parallel edges to %if.then sum to more than 2^32, so
+; scaling the frequency by them must not lose enough precision to round
+; %if.then's probability up to one; that would drop the recorded probability,
+; because probability one needs no annotation.
+;
+; %loop 1 = 1
+; %if.then 21474836475/21474836511 ~ 1
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_common_dest_weight_sum_exceeds_32_bits
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - if.then: float = 1000.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_common_dest_weight_sum_exceeds_32_bits'
+; VPLAN: vector.body:
+; VPLAN-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%{{.+}}>
+; VPLAN-NEXT: EMIT ir<%gep.idx> = getelementptr inbounds ir<%idx>, ir<%iv>
+; VPLAN-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep.idx>
+; VPLAN-NEXT: Successor(s): if.then
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then:
+; VPLAN-NEXT: EMIT vp<[[C0:%.+]]> = icmp eq ir<%l>, ir<0>
+; VPLAN-NEXT: EMIT vp<[[C1:%.+]]> = icmp eq ir<%l>, ir<1>
+; VPLAN-NEXT: EMIT vp<[[C2:%.+]]> = icmp eq ir<%l>, ir<2>
+; VPLAN-NEXT: EMIT vp<[[C3:%.+]]> = icmp eq ir<%l>, ir<3>
+; VPLAN-NEXT: EMIT vp<[[C4:%.+]]> = icmp eq ir<%l>, ir<4>
+; VPLAN-NEXT: EMIT vp<[[OR0:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
+; VPLAN-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[OR0]]>, vp<[[C2]]>
+; VPLAN-NEXT: EMIT vp<[[OR2:%.+]]> = or vp<[[OR1]]>, vp<[[C3]]>
+; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = or vp<[[OR2]]>, vp<[[C4]]>
+; VPLAN-NEXT: EMIT vp<{{.+}}> = not vp<[[MASK]]>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 100%)
+; VPLAN-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %l = load i32, ptr %gep.idx, align 4
+ switch i32 %l, label %latch [
+ i32 0, label %if.then
+ i32 1, label %if.then
+ i32 2, label %if.then
+ i32 3, label %if.then
+ i32 4, label %if.then
+ ], !prof !10
+
+if.then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 1, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @switch_many_edges_to_latch(ptr noalias %a, ptr noalias %idx) {
+; %if.then is reached only via the switch's default edge, while 32 parallel
+; edges go to %latch. BlockFrequencyInfo rounds once per edge, so the
+; cross-check's tolerance has to account for the number of edges, not just the
+; number of blocks.
+;
+; %loop 1 = 1
+; %if.then 1000/1224 ~ 0.8170
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_many_edges_to_latch
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - if.then: float = 816.99,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_many_edges_to_latch'
+; VPLAN: if.then:
+; VPLAN: EMIT store ir<1>, ir<%gep.a>, vp<{{.+}}> (!vplan.execution.probability 81.7%)
+; VPLAN-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %l = load i32, ptr %gep.idx, align 4
+ switch i32 %l, label %if.then [
+ i32 0, label %latch
+ i32 1, label %latch
+ i32 2, label %latch
+ i32 3, label %latch
+ i32 4, label %latch
+ i32 5, label %latch
+ i32 6, label %latch
+ i32 7, label %latch
+ i32 8, label %latch
+ i32 9, label %latch
+ i32 10, label %latch
+ i32 11, label %latch
+ i32 12, label %latch
+ i32 13, label %latch
+ i32 14, label %latch
+ i32 15, label %latch
+ i32 16, label %latch
+ i32 17, label %latch
+ i32 18, label %latch
+ i32 19, label %latch
+ i32 20, label %latch
+ i32 21, label %latch
+ i32 22, label %latch
+ i32 23, label %latch
+ i32 24, label %latch
+ i32 25, label %latch
+ i32 26, label %latch
+ i32 27, label %latch
+ i32 28, label %latch
+ i32 29, label %latch
+ i32 30, label %latch
+ i32 31, label %latch
+ ], !prof !11
+
+if.then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 1, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+!10 = !{!"branch_weights", i32 36, i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295}
+!11 = !{!"branch_weights", i32 1000, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7}
+
!0 = !{!"branch_weights", i32 1, i32 3}
!1 = !{!"branch_weights", i32 1, i32 2}
!2 = !{!"branch_weights", i32 1, i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
index af3ff98adf5d7..cb8fe6f49d926 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
@@ -41,11 +41,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; PREDICATE-NEXT: Successor(s): if.then
; PREDICATE-EMPTY:
; PREDICATE-NEXT: if.then:
-; PREDICATE-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25.00%)
+; PREDICATE-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25%)
; PREDICATE-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; PREDICATE-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; PREDICATE-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; PREDICATE-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25.00%)
+; PREDICATE-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25%)
; PREDICATE-NEXT: Successor(s): latch
; PREDICATE-EMPTY:
; PREDICATE-NEXT: latch:
@@ -84,11 +84,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; CONSTRUCT-NEXT: Successor(s): if.then
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: if.then:
-; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25.00%)
+; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25%)
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; CONSTRUCT-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25.00%)
+; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25%)
; CONSTRUCT-NEXT: Successor(s): latch
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: latch:
@@ -131,7 +131,7 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; REGION-EMPTY:
; REGION-NEXT: <xVFxUF> pred.store: {
; REGION-NEXT: pred.store.entry:
-; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!vplan.execution.probability 25.00%)
+; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!vplan.execution.probability 25%)
; REGION-NEXT: Successor(s): pred.store.if, pred.store.continue
; REGION-EMPTY:
; REGION-NEXT: pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
index 1d84b17d3dd71..0227d7b99c9b6 100644
--- a/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
@@ -10,19 +10,19 @@ define void @predicated_store(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
; VF4IC1: [[SCALAR_PH]]:
@@ -39,19 +39,19 @@ define void @predicated_store(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
+; VF2IC2: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
; VF2IC2: [[SCALAR_PH]]:
@@ -97,19 +97,19 @@ define void @predicated_store_sunk_address(ptr %a, ptr %b, i32 %n) {
; VF4IC1: br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF7]]:
; VF4IC1: [[PRED_STORE_CONTINUE8]]:
-; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF4IC1: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -130,19 +130,19 @@ define void @predicated_store_sunk_address(ptr %a, ptr %b, i32 %n) {
; VF2IC2: br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF8]]:
; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2: br i1 [[TMP25:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -203,7 +203,7 @@ define void @predicated_store_no_profile(ptr %a, i32 %n) {
; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VF4IC1: [[SCALAR_PH]]:
@@ -232,7 +232,7 @@ define void @predicated_store_no_profile(ptr %a, i32 %n) {
; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VF2IC2: [[SCALAR_PH]]:
@@ -287,7 +287,7 @@ define void @predicated_store_unprofiled_predicate(ptr %a, i32 %n) {
; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -316,7 +316,7 @@ define void @predicated_store_unprofiled_predicate(ptr %a, i32 %n) {
; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -358,19 +358,19 @@ define void @predicated_store_almost_always_taken(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
+; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
+; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -387,19 +387,19 @@ define void @predicated_store_almost_always_taken(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
+; VF2IC2: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -443,19 +443,19 @@ define void @predicated_store_non_reciprocal(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF22]]
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF22]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF22]]
+; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF22]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF22]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF22]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -472,19 +472,19 @@ define void @predicated_store_non_reciprocal(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
+; VF2IC2: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF22]]
+; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF22]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF22]]
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF22]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF22]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF22]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -528,19 +528,19 @@ define void @predicated_store_taken_weight_underflow(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF26]]
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF26]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF26]]
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF26]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF26]]
+; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF26]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP14:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -559,19 +559,19 @@ define void @predicated_store_taken_weight_underflow(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
+; VF2IC2: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF26]]
+; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF26]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF26]]
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF26]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF26]]
+; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF26]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
+; VF2IC2: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -618,19 +618,19 @@ define void @predicated_block_with_multiple_ops(ptr noalias %a, ptr noalias %b,
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF7]]
+; VF4IC1: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF7]]
+; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF7]]
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF7]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
+; VF4IC1: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -647,19 +647,19 @@ define void @predicated_block_with_multiple_ops(ptr noalias %a, ptr noalias %b,
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
+; VF2IC2: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF7]]
+; VF2IC2: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF7]]
+; VF2IC2: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF7]]
+; VF2IC2: br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF7]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
+; VF2IC2: br i1 [[TMP24:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -710,19 +710,19 @@ define void @predicated_store_multiple_incoming_edges(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF32]]
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF32]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF32]]
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF32]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF32]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF32]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
+; VF4IC1: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -743,19 +743,19 @@ define void @predicated_store_multiple_incoming_edges(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF32]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF32]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF32]]
+; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF32]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF32]]
+; VF2IC2: br i1 [[TMP24:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF32]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
+; VF2IC2: br i1 [[TMP27:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -822,7 +822,7 @@ define void @predicated_store_multiple_incoming_edges_one_unprofiled(ptr %a, i32
; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
+; VF4IC1: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -855,7 +855,7 @@ define void @predicated_store_multiple_incoming_edges_one_unprofiled(ptr %a, i32
; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
+; VF2IC2: br i1 [[TMP27:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -912,31 +912,31 @@ define void @predicated_store_switch_multiple_cases(ptr %a, i32 %n) {
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF37:![0-9]+]]
+; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF37:![0-9]+]]
; VF4IC1: [[PRED_STORE_IF7]]:
; VF4IC1: [[PRED_STORE_CONTINUE8]]:
-; VF4IC1: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]], !prof [[PROF37]]
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]], !prof [[PROF37]]
; VF4IC1: [[PRED_STORE_IF9]]:
; VF4IC1: [[PRED_STORE_CONTINUE10]]:
-; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]], !prof [[PROF37]]
+; VF4IC1: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]], !prof [[PROF37]]
; VF4IC1: [[PRED_STORE_IF11]]:
; VF4IC1: [[PRED_STORE_CONTINUE12]]:
-; VF4IC1: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]], !prof [[PROF37]]
+; VF4IC1: br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]], !prof [[PROF37]]
; VF4IC1: [[PRED_STORE_IF13]]:
; VF4IC1: [[PRED_STORE_CONTINUE14]]:
-; VF4IC1: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -954,31 +954,31 @@ define void @predicated_store_switch_multiple_cases(ptr %a, i32 %n) {
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
+; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF37:![0-9]+]]
+; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF37:![0-9]+]]
; VF2IC2: [[PRED_STORE_IF8]]:
; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]], !prof [[PROF37]]
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]], !prof [[PROF37]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
-; VF2IC2: br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]], !prof [[PROF37]]
+; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]], !prof [[PROF37]]
; VF2IC2: [[PRED_STORE_IF12]]:
; VF2IC2: [[PRED_STORE_CONTINUE13]]:
-; VF2IC2: br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]], !prof [[PROF37]]
+; VF2IC2: br i1 [[TMP24:%.*]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]], !prof [[PROF37]]
; VF2IC2: [[PRED_STORE_IF14]]:
; VF2IC2: [[PRED_STORE_CONTINUE15]]:
-; VF2IC2: br i1 [[TMP24:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
+; VF2IC2: br i1 [[TMP25:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -1030,19 +1030,19 @@ define void @merged_replicate_regions_inconsistent_weights(ptr noalias %a, ptr n
; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1: [[VECTOR_PH]]:
; VF4IC1: [[VECTOR_BODY:.*]]:
-; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF]]:
; VF4IC1: [[PRED_STORE_CONTINUE]]:
-; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF1]]:
; VF4IC1: [[PRED_STORE_CONTINUE2]]:
-; VF4IC1: br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF3]]:
; VF4IC1: [[PRED_STORE_CONTINUE4]]:
-; VF4IC1: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
; VF4IC1: [[PRED_STORE_IF5]]:
; VF4IC1: [[PRED_STORE_CONTINUE6]]:
-; VF4IC1: br i1 [[TMP16:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1: [[SCALAR_PH]]:
@@ -1062,19 +1062,19 @@ define void @merged_replicate_regions_inconsistent_weights(ptr noalias %a, ptr n
; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2: [[VECTOR_BODY:.*]]:
-; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF]]:
; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
-; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
-; VF2IC2: br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
+; VF2IC2: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2: br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
+; VF2IC2: br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2: [[SCALAR_PH]]:
@@ -1119,6 +1119,91 @@ exit:
ret void
}
+; Predicated store where the condition is taken with a probability just below
+; one. Rounding the probability must not push it to exactly one, which would
+; drop the branch weights entirely, as a block that always executes needs none.
+define void @predicated_store_taken_weight_rounds_to_one(ptr %a, i32 %n) {
+; VF4IC1-LABEL: define void @predicated_store_taken_weight_rounds_to_one(
+; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF4IC1: [[ENTRY:.*:]]
+; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1: [[VECTOR_BODY:.*]]:
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF44:![0-9]+]]
+; VF4IC1: [[PRED_STORE_IF]]:
+; VF4IC1: [[PRED_STORE_CONTINUE]]:
+; VF4IC1: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF44]]
+; VF4IC1: [[PRED_STORE_IF1]]:
+; VF4IC1: [[PRED_STORE_CONTINUE2]]:
+; VF4IC1: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF44]]
+; VF4IC1: [[PRED_STORE_IF3]]:
+; VF4IC1: [[PRED_STORE_CONTINUE4]]:
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF44]]
+; VF4IC1: [[PRED_STORE_IF5]]:
+; VF4IC1: [[PRED_STORE_CONTINUE6]]:
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP45:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
+; VF4IC1: [[SCALAR_PH]]:
+; VF4IC1: [[LOOP:.*]]:
+; VF4IC1: br i1 [[C:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF46:![0-9]+]]
+; VF4IC1: [[IF_THEN]]:
+; VF4IC1: [[LATCH]]:
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP47:![0-9]+]]
+; VF4IC1: [[EXIT]]:
+;
+; VF2IC2-LABEL: define void @predicated_store_taken_weight_rounds_to_one(
+; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF2IC2: [[ENTRY:.*:]]
+; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2: [[VECTOR_BODY:.*]]:
+; VF2IC2: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF44:![0-9]+]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF44]]
+; VF2IC2: [[PRED_STORE_IF2]]:
+; VF2IC2: [[PRED_STORE_CONTINUE3]]:
+; VF2IC2: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF44]]
+; VF2IC2: [[PRED_STORE_IF4]]:
+; VF2IC2: [[PRED_STORE_CONTINUE5]]:
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF44]]
+; VF2IC2: [[PRED_STORE_IF6]]:
+; VF2IC2: [[PRED_STORE_CONTINUE7]]:
+; VF2IC2: br i1 [[TMP15:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP45:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
+; VF2IC2: [[SCALAR_PH]]:
+; VF2IC2: [[LOOP:.*]]:
+; VF2IC2: br i1 [[C:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF46:![0-9]+]]
+; VF2IC2: [[IF_THEN]]:
+; VF2IC2: [[LATCH]]:
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP47:![0-9]+]]
+; VF2IC2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %c = icmp sgt i32 %val, 0
+ br i1 %c, label %if.then, label %latch, !prof !8
+
+if.then:
+ store i32 0, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
!0 = !{!"branch_weights", i32 1, i32 1000}
!1 = !{!"branch_weights", i32 1, i32 7}
!2 = !{!"branch_weights", i32 1, i32 1}
@@ -1127,6 +1212,7 @@ exit:
!5 = !{!"branch_weights", i32 3, i32 2}
!6 = !{!"branch_weights", i32 1, i32 100000}
!7 = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
+!8 = !{!"branch_weights", i32 4294967295, i32 1}
;.
; VF4IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
; VF4IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 7}
@@ -1172,6 +1258,10 @@ exit:
; VF4IC1: [[LOOP41]] = distinct !{[[LOOP41]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF42]] = !{!"branch_weights", i32 1, i32 1000}
; VF4IC1: [[LOOP43]] = distinct !{[[LOOP43]], [[META5]], [[META4]], [[META10]]}
+; VF4IC1: [[PROF44]] = !{!"branch_weights", i32 2147483647, i32 1}
+; VF4IC1: [[LOOP45]] = distinct !{[[LOOP45]], [[META4]], [[META5]], [[META6]]}
+; VF4IC1: [[PROF46]] = !{!"branch_weights", i32 -1, i32 1}
+; VF4IC1: [[LOOP47]] = distinct !{[[LOOP47]], [[META5]], [[META4]], [[META10]]}
;.
; VF2IC2: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
; VF2IC2: [[PROF1]] = !{!"branch_weights", i32 1, i32 7}
@@ -1217,4 +1307,8 @@ exit:
; VF2IC2: [[LOOP41]] = distinct !{[[LOOP41]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF42]] = !{!"branch_weights", i32 1, i32 1000}
; VF2IC2: [[LOOP43]] = distinct !{[[LOOP43]], [[META5]], [[META4]], [[META10]]}
+; VF2IC2: [[PROF44]] = !{!"branch_weights", i32 2147483647, i32 1}
+; VF2IC2: [[LOOP45]] = distinct !{[[LOOP45]], [[META4]], [[META5]], [[META6]]}
+; VF2IC2: [[PROF46]] = !{!"branch_weights", i32 -1, i32 1}
+; VF2IC2: [[LOOP47]] = distinct !{[[LOOP47]], [[META5]], [[META4]], [[META10]]}
;.
>From b09f08007c32320b8e08d74eb55d659a2cd10c27 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 27 Aug 2026 11:05:21 +0100
Subject: [PATCH 7/7] !fixup use frequency, other simplficiations
---
.../Transforms/Vectorize/LoopVectorize.cpp | 58 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 37 ++-
.../Transforms/Vectorize/VPlanPredicator.cpp | 10 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 65 ++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 39 +--
.../Transforms/Vectorize/VPlanTransforms.h | 5 -
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 28 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 89 +++---
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 23 +-
....ll => execution-frequencies-match-bfi.ll} | 254 +++++++++++++++---
.../VPlan/vplan-print-before-after-all.ll | 1 -
.../VPlan/vplan-printing-branch-weights.ll | 12 +-
12 files changed, 401 insertions(+), 220 deletions(-)
rename llvm/test/Transforms/LoopVectorize/VPlan/{execution-probabilities-match-block-frequencies.ll => execution-frequencies-match-bfi.ll} (76%)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1fb84b2158366..3b8899514313f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6453,25 +6453,21 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
static void printOptimizedVPlan(VPlan &) {}
#ifndef NDEBUG
-/// Cross-check the execution probabilities computed by
-/// vputils::computeExecutionProbabilities for the blocks of the loop region of
-/// \p Plan against the frequencies \p BFI computed for the corresponding blocks
-/// of \p OrigLoop.
+/// Cross-check vputils::computeExecutionFrequencies for the loop region of
+/// \p Plan against BlockFrequencyInfo for the blocks of \p OrigLoop.
/// FIXME: Temporary verification aid, to be removed.
-static bool verifyExecutionProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
- LoopInfo *LI,
- BlockFrequencyInfo &BFI) {
- // The verification is limited to inner loops where the latch is the only
- // exiting block and there are no extra VPBBs not mapped to IR BBs (when
- // tail folding).
+static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
+ LoopInfo *LI,
+ LoopVectorizationCostModel &CM) {
+ // Limited to inner loops with the latch as only exiting block and no extra
+ // VPBBs without a matching IR BB (as introduced by tail folding).
if (Plan.isOuterLoop() ||
OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch() ||
Plan.hasTailFolded())
return true;
- // Visit the blocks of the loop region in the same order as
- // introduceMasksAndLinearize does. Both traversals are reverse post-orders of
- // the same CFG, so they visit corresponding blocks at the same index.
+ // Visit the region's blocks in the same order as introduceMasksAndLinearize.
+ // Both are reverse post-orders of the same CFG, so indices correspond.
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion()->getEntryBasicBlock());
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
@@ -6481,36 +6477,34 @@ static bool verifyExecutionProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
LoopBlocksRPO OrigRPO(OrigLoop);
OrigRPO.perform(LI);
+ // Only request the expensive BFI once the cheap bail-outs are past.
+ BlockFrequencyInfo &BFI = CM.getBFI();
uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
if (HeaderFreq == 0)
return true;
- // BFI's fixed-point mass propagation rounds once per edge, losing up to 1 ULP
- // per edge on the paths reaching a block. Bound that by the total number of
- // edges in the region, as a block distributing its frequency over many
- // successors rounds once for each of them.
+ // BFI's fixed-point mass propagation loses up to 1 ULP per edge, so bound the
+ // error by the number of edges in the region.
uint64_t Edges = 0;
for (const VPBasicBlock *VPBB : Blocks)
Edges += VPBB->getNumSuccessors();
uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
- DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities =
- vputils::computeExecutionProbabilities(Blocks);
+ DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies =
+ vputils::computeExecutionFrequencies(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
- // Currently VPlan-based probabilities are only computed when all blocks
- // have branch-weights. Compare at BranchProbability's coarser resolution,
- // which is as precise as BFI's frequencies get.
- VPExecutionProbability Prob = Probabilities.lookup(VPBB);
- if (Prob.isUnknown())
+ // Compare at BranchProbability's coarser resolution, which is as precise as
+ // BFI's frequencies get.
+ std::optional<BlockFrequency> Freq = Frequencies.lookup(VPBB);
+ if (!Freq)
continue;
- BranchProbability Computed = Prob;
+ BranchProbability Computed = vputils::getExecutionProbability(*Freq);
- // Clamp the frequency to the header's; it may exceed it slightly due to
- // BFI's rounding.
- uint64_t Freq = BFI.getBlockFreq(BB).getFrequency();
+ // Clamp to the header's frequency, which BFI's rounding may exceed.
+ uint64_t BBFreq = BFI.getBlockFreq(BB).getFrequency();
BranchProbability Expected = BranchProbability::getBranchProbability(
- std::min(Freq, HeaderFreq), HeaderFreq);
+ std::min(BBFreq, HeaderFreq), HeaderFreq);
if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
Tolerance)
continue;
@@ -6609,9 +6603,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
if (CM.foldTailByMasking())
RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
- assert(verifyExecutionProbabilitiesMatchBFI(*VPlan0, OrigLoop, LI,
- CM.getBFI()) &&
- "execution probabilities do not match the loop's frequencies");
+ assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, CM) &&
+ "execution frequencies do not match the loop's block frequencies");
RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
return VPlan0;
@@ -6828,7 +6821,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
- RUN_VPLAN_PASS(VPlanTransforms::dropUnguardedExecutionProbabilities, *Plan);
RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
Range);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f25b152a84da2..5c0ab5a8cf76f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -38,12 +38,13 @@
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/FMF.h"
#include "llvm/IR/Operator.h"
-#include "llvm/Support/BranchProbability.h"
+#include "llvm/Support/BlockFrequency.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/InstructionCost.h"
#include <cassert>
#include <cstddef>
#include <functional>
+#include <optional>
#include <string>
#include <utility>
#include <variant>
@@ -74,10 +75,6 @@ struct VPCostContext;
using VPlanPtr = std::unique_ptr<VPlan>;
-/// The probability with which a block or recipe executes, relative to the entry
-/// of the loop region, which always executes.
-using VPExecutionProbability = BranchProbability;
-
/// \enum UncountableExitStyle
/// Different methods of handling early exits.
///
@@ -1185,14 +1182,12 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
class LLVM_ABI_FOR_TEST VPIRMetadata {
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- /// Name of the VPlan-internal metadata kind recording the probability with
- /// which a recipe executes; see setExecutionProbability.
- static constexpr StringLiteral ExecutionProbabilityMDName =
- "vplan.execution.probability";
+ /// Name of the VPlan-internal metadata kind holding the execution frequency.
+ static constexpr StringLiteral ExecutionFrequencyMDName =
+ "vplan.execution.frequency";
- /// Returns the ID of the metadata kind named \p Kind. Any attached node
- /// provides the context the IDs are assigned in, as all nodes belong to the
- /// context of the VPlan's function.
+ /// Returns the ID of the metadata kind named \p Kind, taking the context from
+ /// any attached node; all belong to the context of the VPlan's function.
unsigned getMDKindID(StringRef Kind) const {
assert(!Metadata.empty() && "no node to take the context from");
return Metadata.front().second->getContext().getMDKindID(Kind);
@@ -1206,7 +1201,7 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
VPIRMetadata(Instruction &I) {
getMetadataToPropagate(&I, Metadata);
// Retain the branch weights of terminators. They are used to compute the
- // probabilities with which the blocks of the original loop execute.
+ // frequencies with which the blocks of the original loop execute.
if (I.isTerminator())
if (MDNode *BW = I.getMetadata(LLVMContext::MD_prof))
Metadata.emplace_back(LLVMContext::MD_prof, BW);
@@ -1244,16 +1239,16 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
return It != Metadata.end() ? It->second : nullptr;
}
- /// Record that the recipe executes with probability \p Prob, relative to the
- /// entry of the loop region.
- void setExecutionProbability(VPExecutionProbability Prob, LLVMContext &Ctx);
+ /// Record that the recipe executes with frequency \p Freq, relative to the
+ /// entry of the loop region; see vputils::AlwaysExecutesFreq.
+ void setExecutionFrequency(std::optional<BlockFrequency> Freq,
+ LLVMContext &Ctx);
- /// Returns the probability recorded by setExecutionProbability, or an unknown
- /// probability if none has been recorded.
- VPExecutionProbability getExecutionProbability() const;
+ /// Returns the frequency recorded by setExecutionFrequency, if any.
+ std::optional<BlockFrequency> getExecutionFrequency() const;
- /// Drop the probability recorded by setExecutionProbability, if any.
- void clearExecutionProbability();
+ /// Drop the frequency recorded by setExecutionFrequency, if any.
+ void clearExecutionFrequency();
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print metadata with node IDs.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index c34f59431b14e..cbc41e82fa87c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -406,8 +406,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Header);
// Non-outer regions with VPBBs only are supported at the moment.
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities =
- vputils::computeExecutionProbabilities(Blocks);
+ DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies =
+ vputils::computeExecutionFrequencies(Blocks);
VPPredicator Predicator(Plan);
for (VPBasicBlock *VPBB : Blocks) {
@@ -421,16 +421,16 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
if (!BlockMask)
continue;
- // Mask all VPInstructions in the block and record the probability with
+ // Mask all VPInstructions in the block and record the frequency with
// which the masked recipes execute.
- VPExecutionProbability Prob = Probabilities.lookup(VPBB);
+ std::optional<BlockFrequency> Freq = Frequencies.lookup(VPBB);
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
continue;
VPI->addMask(BlockMask);
if (VPI->isMasked())
- VPI->setExecutionProbability(Prob, Plan.getContext());
+ VPI->setExecutionFrequency(Freq, Plan.getContext());
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f4d48c6481032..996fbd6f2edef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2129,44 +2129,50 @@ void VPIRPhi::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
void VPIRMetadata::applyMetadata(Instruction &I) const {
- assert(getExecutionProbability().isUnknown() &&
- "VPlan-internal metadata must not be propagated to IR");
+ if (Metadata.empty())
+ return;
+ // The execution frequency is VPlan-internal and must not reach IR; recipes
+ // that end up unguarded may still carry one.
+ unsigned InternalKind = getMDKindID(ExecutionFrequencyMDName);
for (const auto &[Kind, Node] : Metadata)
- I.setMetadata(Kind, Node);
+ if (Kind != InternalKind)
+ I.setMetadata(Kind, Node);
}
-/// Returns the execution probability recorded in \p Node.
-static VPExecutionProbability getExecutionProbabilityFromMD(const MDNode *Node) {
- return VPExecutionProbability::getRaw(
- mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue());
+/// Returns the execution frequency recorded in \p Node.
+static BlockFrequency getExecutionFrequencyFromMD(const MDNode *Node) {
+ uint64_t Freq =
+ mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue();
+ assert(Freq <= vputils::AlwaysExecutesFreq &&
+ "frequency cannot exceed the one of an always executing block");
+ return BlockFrequency(Freq);
}
-void VPIRMetadata::setExecutionProbability(VPExecutionProbability Prob,
- LLVMContext &Ctx) {
+void VPIRMetadata::setExecutionFrequency(std::optional<BlockFrequency> Freq,
+ LLVMContext &Ctx) {
// A recipe that never or always executes needs no annotation.
- if (Prob.isUnknown() || Prob.isZero() || Prob.isOne())
+ if (!Freq || Freq->getFrequency() == 0 ||
+ Freq->getFrequency() == vputils::AlwaysExecutesFreq)
return;
- // The numerator is relative to BranchProbability's fixed denominator and is
- // less than it, as probability one has been excluded above.
- Constant *Numerator =
- ConstantInt::get(Type::getInt32Ty(Ctx), Prob.getNumerator());
- setMetadata(Ctx.getMDKindID(ExecutionProbabilityMDName),
- MDNode::get(Ctx, {ConstantAsMetadata::get(Numerator)}));
+ Constant *Frequency =
+ ConstantInt::get(Type::getInt64Ty(Ctx), Freq->getFrequency());
+ setMetadata(Ctx.getMDKindID(ExecutionFrequencyMDName),
+ MDNode::get(Ctx, {ConstantAsMetadata::get(Frequency)}));
}
-VPExecutionProbability VPIRMetadata::getExecutionProbability() const {
+std::optional<BlockFrequency> VPIRMetadata::getExecutionFrequency() const {
if (Metadata.empty())
- return VPExecutionProbability::getUnknown();
- MDNode *Node = getMetadata(getMDKindID(ExecutionProbabilityMDName));
+ return std::nullopt;
+ MDNode *Node = getMetadata(getMDKindID(ExecutionFrequencyMDName));
if (!Node)
- return VPExecutionProbability::getUnknown();
- return getExecutionProbabilityFromMD(Node);
+ return std::nullopt;
+ return getExecutionFrequencyFromMD(Node);
}
-void VPIRMetadata::clearExecutionProbability() {
+void VPIRMetadata::clearExecutionFrequency() {
if (Metadata.empty())
return;
- unsigned ID = getMDKindID(ExecutionProbabilityMDName);
+ unsigned ID = getMDKindID(ExecutionFrequencyMDName);
erase_if(Metadata, [ID](const auto &P) { return P.first == ID; });
}
@@ -2203,14 +2209,11 @@ void VPIRMetadata::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
O << "{";
interleaveComma(Weights, O);
O << "}";
- } else if (MDNames[Kind] == ExecutionProbabilityMDName) {
- // Print execution probabilities as percentages, which relates them to
- // the block frequencies of the original loop. Use %g with a few
- // significant digits, so that the tiny probabilities of deeply nested
- // blocks stay legible without adding noise to the common ones.
- O << format("%.4g%%",
- 100.0 * getExecutionProbabilityFromMD(Node).getNumerator() /
- VPExecutionProbability::getDenominator());
+ } else if (MDNames[Kind] == ExecutionFrequencyMDName) {
+ // Print the frequency together with the probability it corresponds to.
+ uint64_t Freq = getExecutionFrequencyFromMD(Node).getFrequency();
+ O << Freq
+ << format(" (%.4g%%)", 100.0 * Freq / vputils::AlwaysExecutesFreq);
} else {
Node->printAsOperand(O, M);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e0b287aae1448..118fc6404c51f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -485,13 +485,13 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
continue;
// The merged region is entered whenever either of the original regions was,
- // so use the higher, i.e. more conservative, of their entry probabilities.
+ // so use the higher, i.e. more conservative, of their entry frequencies.
VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
- VPExecutionProbability Prob1 =
- Region1->getEntryBranchOnMask()->getExecutionProbability();
- VPExecutionProbability Prob2 = Guard2->getExecutionProbability();
- if (!Prob1.isUnknown() && !Prob2.isUnknown() && Prob2 < Prob1)
- Guard2->setExecutionProbability(Prob1, Plan.getContext());
+ std::optional<BlockFrequency> Freq1 =
+ Region1->getEntryBranchOnMask()->getExecutionFrequency();
+ std::optional<BlockFrequency> Freq2 = Guard2->getExecutionFrequency();
+ if (Freq1 && Freq2 && *Freq2 < *Freq1)
+ Guard2->setExecutionFrequency(Freq1, Plan.getContext());
// Note: No fusion-preventing memory dependencies are expected in either
// region. Such dependencies should be rejected during earlier dependence
@@ -561,10 +561,10 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
PredRecipe->getDebugLoc());
// The predicated recipe executes exactly when the guarding branch-on-mask is
- // taken, so move its execution probability there.
- BOMRecipe->setExecutionProbability(
- RecipeWithoutMask->getExecutionProbability(), Plan.getContext());
- RecipeWithoutMask->clearExecutionProbability();
+ // taken, so move its execution frequency there.
+ BOMRecipe->setExecutionFrequency(RecipeWithoutMask->getExecutionFrequency(),
+ Plan.getContext());
+ RecipeWithoutMask->clearExecutionFrequency();
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
auto *Exiting = Plan.createVPBasicBlock(Twine(RegionName) + ".continue");
@@ -3728,9 +3728,9 @@ static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
VPIRMetadata CommonMetadata = *Recipes.front();
for (VPReplicateRecipe *Recipe : drop_begin(Recipes))
CommonMetadata.intersect(*Recipe);
- // The recipe the common metadata is used for is not predicated, so it does
- // not execute with the probability shared by the recipes in the group.
- CommonMetadata.clearExecutionProbability();
+ // The recipe using the common metadata is not predicated, so it does not
+ // share the group's execution frequency.
+ CommonMetadata.clearExecutionFrequency();
return CommonMetadata;
}
@@ -5595,19 +5595,6 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
}
}
-void VPlanTransforms::dropUnguardedExecutionProbabilities(VPlan &Plan) {
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- if (RepR && RepR->isPredicated())
- continue;
- if (auto *MD = dyn_cast<VPIRMetadata>(&R))
- MD->clearExecutionProbability();
- }
- }
-}
-
/// Returns true if \p Info's parameter kinds are compatible with \p Args.
static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
PredicatedScalarEvolution &PSE, const Loop *L) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 497168277604a..c3dd00acfe839 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -617,11 +617,6 @@ struct VPlanTransforms {
/// enabled by prior widening of consecutive memory operations for now.
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range);
- /// Drop the execution probabilities recorded on all recipes that will not be
- /// guarded by a branch. Currently those are all recipes except predicated
- /// VPReplicateRecipes.
- static void dropUnguardedExecutionProbabilities(VPlan &Plan);
-
/// Convert call VPInstructions in \p Plan into widened call, vector
/// intrinsic or replicate recipes based on a cost comparison via \p CostCtx.
static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index aa50857834cfe..b50b8894091c7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -676,20 +676,19 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
return New;
}
-/// Converts the probability \p Prob with which a block is entered to the branch
-/// weights of the conditional branch guarding it, or returns nullptr if \p Prob
-/// is unknown.
-static MDNode *convertProbabilityToBranchWeights(VPExecutionProbability Prob,
- LLVMContext &Ctx) {
- if (Prob.isUnknown())
+/// Converts the frequency \p Freq with which a block is entered to branch
+/// weights for the branch guarding it, or nullptr if \p Freq is unknown.
+static MDNode *
+convertFrequencyToBranchWeights(std::optional<BlockFrequency> Freq,
+ LLVMContext &Ctx) {
+ if (!Freq)
return nullptr;
- BranchProbability P = Prob;
+ BranchProbability P = vputils::getExecutionProbability(*Freq);
// Use the numerators of P and its complement as weights and reduce them via
- // gcd to keep them small. Round a probability that underflows P up, to keep a
- // rarely entered block distinguishable from a never entered one.
- uint32_t Taken = std::max(P.getNumerator(), 1u);
- uint32_t NotTaken = BranchProbability::getDenominator() - Taken;
+ // gcd to keep them small. Neither is zero, as P is neither zero nor one.
+ uint32_t Taken = P.getNumerator();
+ uint32_t NotTaken = P.getCompl().getNumerator();
uint32_t GCD = std::gcd(Taken, NotTaken);
return MDBuilder(Ctx).createBranchWeights(Taken / GCD, NotTaken / GCD);
}
@@ -743,12 +742,11 @@ static void convertRecipesInRegionBlocksToSingleScalar(VPlan &Plan, Type *IdxTy,
RepR->replaceAllUsesWith(NewR);
RepR->eraseFromParent();
} else if (auto *BranchOnMask = dyn_cast<VPBranchOnMaskRecipe>(&OldR)) {
- // Turn the probability with which the predicated block is entered into
- // branch weights on the generated conditional branch.
+ // Turn the frequency of the predicated block into branch weights.
auto *BOC = Builder.createNaryOp(VPInstruction::BranchOnCond,
{BranchOnMask->getOperand(0)}, OldDL);
- if (MDNode *Weights = convertProbabilityToBranchWeights(
- BranchOnMask->getExecutionProbability(), Plan.getContext()))
+ if (MDNode *Weights = convertFrequencyToBranchWeights(
+ BranchOnMask->getExecutionFrequency(), Plan.getContext()))
BOC->setMetadata(LLVMContext::MD_prof, Weights);
BranchOnMask->eraseFromParent();
} else if (auto *PredPhi = dyn_cast<VPPredInstPHIRecipe>(&OldR)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 373209697d601..0d819cdfd9b29 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -13,7 +13,6 @@
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/TypeSwitch.h"
@@ -1118,6 +1117,23 @@ SmallVector<VPUser *> vputils::collectUsersRecursively(VPValue *V) {
return Users.takeVector();
}
+/// Returns \p Num / \p Denom as a BranchProbability, clamped so a ratio that is
+/// neither zero nor one does not round to zero or one. BlockFrequencyInfo also
+/// keeps a zero-weight edge distinguishable from an unreachable one.
+static BranchProbability getBranchProbabilityKeepingPartial(uint64_t Num,
+ uint64_t Denom) {
+ BranchProbability P = BranchProbability::getBranchProbability(Num, Denom);
+ if (Num == 0 || Num == Denom)
+ return P;
+ return BranchProbability::getRaw(std::clamp(
+ P.getNumerator(), 1u, BranchProbability::getDenominator() - 1));
+}
+
+BranchProbability vputils::getExecutionProbability(BlockFrequency Freq) {
+ return getBranchProbabilityKeepingPartial(Freq.getFrequency(),
+ AlwaysExecutesFreq);
+}
+
/// Returns the probability of reaching each unique successor of \p VPBB, taken
/// from the branch weights recorded on its terminator, or unknown if not
/// available. See llvm::getBranchProbability in
@@ -1126,8 +1142,8 @@ static SmallVector<std::pair<const VPBasicBlock *, BranchProbability>, 2>
getSuccessorProbabilities(const VPBasicBlock *VPBB) {
ArrayRef<VPBlockBase *> Successors = VPBB->getSuccessors();
// With a single successor the edge is always taken and needs no weights.
- if (Successors.size() == 1)
- return {{cast<VPBasicBlock>(Successors[0]), BranchProbability::getOne()}};
+ if (VPBlockBase *Succ = VPBB->getSingleSuccessor())
+ return {{cast<VPBasicBlock>(Succ), BranchProbability::getOne()}};
// Take the branch weights off the terminator. Without usable weights all
// successors have unknown probability; zero the weights, so the accumulation
@@ -1150,54 +1166,49 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
auto [Succ, Weight] = SuccWeight;
if (Total == 0)
return std::make_pair(Succ, BranchProbability::getUnknown());
- BranchProbability P =
- BranchProbability::getBranchProbability(Weight, Total);
- // A total wider than 32 bits gets scaled down, truncating the numerator.
- // That can collapse a taken edge to probability zero, or a partially taken
- // one to probability one; keep both ends apart, as BlockFrequencyInfo also
- // gives a zero-weight edge a non-zero share.
- if (P.isZero() && Weight != 0)
- P = BranchProbability::getRaw(1);
- else if (P.isOne() && Weight != Total)
- P = BranchProbability::getRaw(BranchProbability::getDenominator() - 1);
- return std::make_pair(Succ, P);
+ return std::make_pair(Succ,
+ getBranchProbabilityKeepingPartial(Weight, Total));
});
}
-DenseMap<const VPBasicBlock *, VPExecutionProbability>
-vputils::computeExecutionProbabilities(ArrayRef<VPBasicBlock *> Blocks) {
+/// Returns \p Freq scaled by \p Prob, rounding up to 1 instead of 0 to keep a
+/// rarely executed block distinguishable from an unreachable one.
+static BlockFrequency scaleKeepingNonZero(BlockFrequency Freq,
+ BranchProbability Prob) {
+ BlockFrequency Scaled = Freq * Prob;
+ if (Scaled == BlockFrequency() && Freq != BlockFrequency() && !Prob.isZero())
+ return BlockFrequency(1);
+ return Scaled;
+}
+
+DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>>
+vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
assert(!Blocks.empty() && "expected at least the header block");
- // Push each block's probability along its outgoing edges, accumulating it in
- // the successors. Blocks is in reverse post-order and the blocks form a DAG
- // (the backedge of a loop region is implicit), so all incoming edges of a
- // block have contributed by the time it is visited and its probability is
- // final.
- DenseMap<const VPBasicBlock *, VPExecutionProbability> Probabilities;
- Probabilities.reserve(Blocks.size());
- // The header (first block) always executes.
- Probabilities[Blocks.front()] = VPExecutionProbability::getOne();
+ // Push each block's frequency along its outgoing edges. Blocks is a DAG in
+ // reverse post-order (the loop region's backedge is implicit), so a block's
+ // frequency is final by the time it is visited.
+ DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies;
+ Frequencies.reserve(Blocks.size());
+ // The header (first block) always executes, the others start out unreachable.
+ Frequencies[Blocks.front()] = BlockFrequency(AlwaysExecutesFreq);
for (VPBasicBlock *VPBB : Blocks.drop_front())
- Probabilities[VPBB] = VPExecutionProbability::getZero();
+ Frequencies[VPBB] = BlockFrequency();
for (VPBasicBlock *VPBB : Blocks) {
- VPExecutionProbability SrcProb = Probabilities.at(VPBB);
+ std::optional<BlockFrequency> SrcFreq = Frequencies.at(VPBB);
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- VPExecutionProbability &SuccProb = Probabilities.at(Succ);
- // An unknown edge or predecessor poisons the successor: its probability
- // is only known if all edges on paths reaching it carry branch weights.
- if (SrcProb.isUnknown() || EdgeProb.isUnknown() || SuccProb.isUnknown()) {
- SuccProb = VPExecutionProbability::getUnknown();
+ std::optional<BlockFrequency> &SuccFreq = Frequencies.at(Succ);
+ // An unknown edge or predecessor poisons the successor.
+ if (!SrcFreq || EdgeProb.isUnknown() || !SuccFreq) {
+ SuccFreq = std::nullopt;
continue;
}
- VPExecutionProbability Contribution = SrcProb * EdgeProb;
- // Force to the lowest possible probability if the product gets rounded to
- // zero, to keep reachable blocks distinguishable from unreachable ones.
- if (Contribution.isZero() && !SrcProb.isZero() && !EdgeProb.isZero())
- Contribution = VPExecutionProbability::getRaw(1);
- SuccProb += Contribution;
+ // The sum can only exceed AlwaysExecutesFreq by rounding.
+ SuccFreq = std::min(BlockFrequency(AlwaysExecutesFreq),
+ *SuccFreq + scaleKeepingNonZero(*SrcFreq, EdgeProb));
}
}
- return Probabilities;
+ return Frequencies;
}
VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 85d286311cab7..ac356f403c135 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -10,6 +10,8 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANUTILS_H
#include "VPlan.h"
+#include "llvm/Support/BlockFrequency.h"
+#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
@@ -221,13 +223,20 @@ SmallVector<VPUser *> collectUsersRecursively(VPValue *V);
VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
const DataLayout &DL);
-/// Computes for each block in \p Blocks the probability that it executes,
-/// relative to the first block in \p Blocks (the header block), which always
-/// executes. \p Blocks must be in reverse post-order. The probability of a
-/// block is the accumulated probability of its incoming edges, or unknown if
-/// any edge on a path reaching it lacks branch weights.
-DenseMap<const VPBasicBlock *, VPExecutionProbability>
-computeExecutionProbabilities(ArrayRef<VPBasicBlock *> Blocks);
+/// Denominator of the frequencies computed by computeExecutionFrequencies, i.e.
+/// the frequency of a block that always executes. Wider than
+/// BranchProbability's 31-bit one, which truncates rarely executed blocks to 0.
+inline constexpr uint64_t AlwaysExecutesFreq = 1ULL << 63;
+
+/// Returns \p Freq as a BranchProbability, relative to AlwaysExecutesFreq.
+BranchProbability getExecutionProbability(BlockFrequency Freq);
+
+/// Computes for each block in \p Blocks, which must be in reverse post-order,
+/// the frequency with which it executes relative to the first (header) block.
+/// The frequency of a block is the sum over its incoming edges, or std::nullopt
+/// if any edge on a path reaching it lacks branch weights.
+DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>>
+computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks);
namespace detail {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
similarity index 76%
rename from llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
rename to llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
index 1a6290cba5f19..0b0a698b42bf2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/execution-probabilities-match-block-frequencies.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
@@ -4,12 +4,12 @@
; RUN: -vplan-print-after=introduceMasksAndLinearize -disable-output %s 2>&1 \
; RUN: | FileCheck --check-prefix=VPLAN %s
-; Check that the execution probabilities VPlan records on the masked recipes of
+; Check that the execution frequencies VPlan records on the masked recipes of
; a block match the block frequencies BlockFrequencyInfo computes for the
; corresponding block of the original scalar loop.
define void @single_pred(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
-; Execution probability of each block of the loop
+; Execution frequency of each block of the loop
;
; %loop 1000/1000 = 1
; %if.then 250/1000 = 1/4
@@ -34,7 +34,7 @@ define void @single_pred(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.frequency 2305843009213693952 (25%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -68,7 +68,7 @@ exit:
}
define void @two_preds(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %idx) {
-; Execution probability of each block of the loop. %merge is reached from both
+; Execution frequency of each block of the loop. %merge is reached from both
; %then (1/4) and %else (3/4 * 1/3 = 1/4).
;
; %loop 1000/1000 = 1
@@ -97,20 +97,20 @@ define void @two_preds(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noali
; VPLAN-NEXT: else:
; VPLAN-NEXT: EMIT vp<[[NOT_C0:%.+]]> = not ir<%c.0>
; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!vplan.execution.probability 75%)
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!vplan.execution.probability 75%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.c>, vp<[[NOT_C0]]> (!vplan.execution.frequency 6917529027641081856 (75%))
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<-100>, vp<[[NOT_C0]]> (!vplan.execution.frequency 6917529027641081856 (75%))
; VPLAN-NEXT: Successor(s): then
; VPLAN-EMPTY:
; VPLAN-NEXT: then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.probability 25%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.frequency 2305843009213693952 (25%))
; VPLAN-NEXT: Successor(s): merge
; VPLAN-EMPTY:
; VPLAN-NEXT: merge:
; VPLAN-NEXT: EMIT vp<[[AND:%.+]]> = logical-and vp<[[NOT_C0]]>, ir<%c.1>
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = or vp<[[AND]]>, ir<%c.0>
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!vplan.execution.probability 50%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, vp<[[MASK]]> (!vplan.execution.frequency 4611686019501129728 (50%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -153,7 +153,7 @@ exit:
}
define void @nested_ifs(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
-; Execution probability of each block of the loop
+; Execution frequency of each block of the loop
;
; %loop 1000/1000 = 1
; %if.0 250/1000 = 1/4
@@ -178,14 +178,14 @@ define void @nested_ifs(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; VPLAN-EMPTY:
; VPLAN-NEXT: if.0:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.probability 25%)
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!vplan.execution.probability 25%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp slt ir<%i>, ir<100>, ir<%c.0> (!vplan.execution.frequency 2305843009213693952 (25%))
; VPLAN-NEXT: Successor(s): if.1
; VPLAN-EMPTY:
; VPLAN-NEXT: if.1:
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = logical-and ir<%c.0>, ir<%c.1>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 12.5%)
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.frequency 1152921504606846976 (12.5%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -222,8 +222,73 @@ exit:
ret void
}
+define void @second_branch_without_weights(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; %merge's branch has no weights, so everything it reaches is unknown and stays
+; unannotated. %merge itself is still known, as both of its incoming edges are.
+;
+; %loop 1000/1000 = 1
+; %if.then.1 250/1000 = 1/4
+; %merge 1000/1000 = 1
+; %if.then.2 unknown
+; %latch unknown
+;
+; BFI-LABEL: block-frequency-info: second_branch_without_weights
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - if.then.1: float = 250.0,
+; BFI-NEXT: - merge: float = 1000.0,
+; BFI-NEXT: - if.then.2: float = 625.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'second_branch_without_weights'
+; VPLAN: if.then.1:
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>, ir<%c.0> (!vplan.execution.frequency 2305843009213693952 (25%))
+; VPLAN-NEXT: Successor(s): merge
+; VPLAN-EMPTY:
+; VPLAN-NEXT: merge:
+; VPLAN-NEXT: Successor(s): if.then.2
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then.2:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0>{{$}}
+; VPLAN-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %i = load i32, ptr %gep.idx, align 4
+ %c.0 = icmp sgt i32 %i, 0
+ br i1 %c.0, label %if.then.1, label %merge, !prof !0
+
+if.then.1:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %i, ptr %gep.a, align 4
+ br label %merge
+
+merge:
+ br i1 %c.0, label %if.then.2, label %latch
+
+if.then.2:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %i, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
define void @switch_common_dest(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %idx) {
-; Execution probability of each block of the loop. %if.then is reached from 2 of
+; Execution frequency of each block of the loop. %if.then is reached from 2 of
; the switch's cases (125 + 250 = 375) and %default via the default edge (500).
;
; %loop 1000/1000 = 1
@@ -256,17 +321,17 @@ define void @switch_common_dest(ptr noalias %a, ptr noalias %b, ptr noalias %c,
; VPLAN-NEXT: EMIT vp<[[ANY:%.+]]> = or vp<[[C0_OR_C1]]>, vp<[[C2]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[ANY]]>
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!vplan.execution.probability 12.5%)
+; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.b>, vp<[[C2]]> (!vplan.execution.frequency 1152921504606846976 (12.5%))
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 37.5%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.frequency 3458764513820540928 (37.5%))
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!vplan.execution.probability 50%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c>, vp<[[DEFAULT]]> (!vplan.execution.frequency 4611686018427387904 (50%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -341,12 +406,12 @@ define void @switch_common_dest_weight_sum_not_a_power_of_two(ptr noalias %a, pt
; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 66.67%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.frequency 6148914689804861440 (66.67%))
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.probability 33.33%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.frequency 3074457347049914368 (33.33%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -414,12 +479,12 @@ define void @switch_common_dest_almost_always_taken(ptr noalias %a, ptr noalias
; VPLAN-NEXT: EMIT vp<[[C0_OR_C1:%.+]]> = or vp<[[C0]]>, vp<[[C1]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[C0_OR_C1]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.probability 100%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[C0_OR_C1]]> (!vplan.execution.frequency 9223372032559808512 (100%))
; VPLAN-NEXT: Successor(s): default
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.probability 4.657e-08%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.frequency 4294967296 (4.657e-08%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -459,9 +524,9 @@ exit:
define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
; %mid is almost never executed, and %if.then is reached from 4 of the switch's
-; cases in %mid. Both probabilities are far below BranchProbability's 2^-31
-; resolution, so they must be represented as block frequencies to stay
-; distinguishable from zero.
+; cases in %mid. Both are far below BranchProbability's 2^-31 resolution, so
+; they must be represented as block frequencies to stay distinguishable from
+; zero.
;
; %loop 1 = 1
; %mid 1/2147483648 ~ 0
@@ -486,7 +551,7 @@ define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %
; VPLAN-EMPTY:
; VPLAN-NEXT: mid:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 4.657e-08%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.frequency 4294967296 (4.657e-08%))
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
@@ -501,7 +566,7 @@ define void @switch_common_dest_almost_never_taken(ptr noalias %a, ptr noalias %
; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 4.657e-08%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.frequency 3435973836 (3.725e-08%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -570,7 +635,7 @@ define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, pt
; VPLAN-EMPTY:
; VPLAN-NEXT: mid:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.probability 4.657e-08%)
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, ir<%c> (!vplan.execution.frequency 4294967296 (4.657e-08%))
; VPLAN-NEXT: Successor(s): if.then
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
@@ -578,7 +643,7 @@ define void @switch_common_dest_many_edges_almost_never_taken(ptr noalias %a, pt
; VPLAN-NEXT: EMIT vp<[[NOT_MASK:%.+]]> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT vp<[[DEFAULT:%.+]]> = logical-and ir<%c>, vp<[[NOT_MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 4.657e-08%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.frequency 3817748708 (4.139e-08%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -626,8 +691,8 @@ exit:
define void @switch_common_dest_weight_sum_exceeds_32_bits(ptr noalias %a, ptr noalias %idx) {
; The weights of the five parallel edges to %if.then sum to more than 2^32, so
; scaling the frequency by them must not lose enough precision to round
-; %if.then's probability up to one; that would drop the recorded probability,
-; because probability one needs no annotation.
+; %if.then's frequency up to the one of an always executing block; that would
+; drop the recorded frequency, because such a frequency needs no annotation.
;
; %loop 1 = 1
; %if.then 21474836475/21474836511 ~ 1
@@ -659,7 +724,7 @@ define void @switch_common_dest_weight_sum_exceeds_32_bits(ptr noalias %a, ptr n
; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = or vp<[[OR2]]>, vp<[[C4]]>
; VPLAN-NEXT: EMIT vp<{{.+}}> = not vp<[[MASK]]>
; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.probability 100%)
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.frequency 9223372023969873920 (100%))
; VPLAN-NEXT: Successor(s): latch
;
entry:
@@ -710,7 +775,7 @@ define void @switch_many_edges_to_latch(ptr noalias %a, ptr noalias %idx) {
;
; VPLAN-LABEL: VPlan for loop in 'switch_many_edges_to_latch'
; VPLAN: if.then:
-; VPLAN: EMIT store ir<1>, ir<%gep.a>, vp<{{.+}}> (!vplan.execution.probability 81.7%)
+; VPLAN: EMIT store ir<1>, ir<%gep.a>, vp<{{.+}}> (!vplan.execution.frequency 7535434672457646080 (81.7%))
; VPLAN-NEXT: Successor(s): latch
;
entry:
@@ -772,6 +837,132 @@ exit:
!10 = !{!"branch_weights", i32 36, i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295, i32 4294967295}
!11 = !{!"branch_weights", i32 1000, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7}
+define void @switch_weights_clamped_at_both_ends(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
+; The weights sum to more than 2^32.
+;
+; %loop 1 = 1
+; %if.then 8589934593/8589934595 ~ 1
+; %default 2/8589934595 ~ 2.33e-10
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: switch_weights_clamped_at_both_ends
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - default: float = 0.00000046566,
+; BFI-NEXT: - if.then: float = 1000.0,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'switch_weights_clamped_at_both_ends'
+; VPLAN: if.then:
+; VPLAN: EMIT vp<[[DEFAULT:%.+]]> = not vp<[[MASK:%.+]]>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.frequency 9223372032559808512 (100%))
+; VPLAN-NEXT: Successor(s): default
+; VPLAN-EMPTY:
+; VPLAN-NEXT: default:
+; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.b>, vp<[[DEFAULT]]> (!vplan.execution.frequency 4294967296 (4.657e-08%))
+; VPLAN-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %l = load i32, ptr %gep.idx, align 4
+ switch i32 %l, label %default [
+ i32 0, label %if.then
+ i32 1, label %if.then
+ i32 2, label %if.then
+ ], !prof !12
+
+default:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 0, ptr %gep.b, align 4
+ br label %latch
+
+if.then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 1, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @nested_blocks_almost_never_entered(ptr noalias %a, ptr noalias %idx) {
+; Each nested block is entered with probability 2^-31
+;
+; %loop 1 = 1
+; %if.then.1 2^-31 ~ 4.66e-10
+; %if.then.2 2^-62 ~ 2.17e-19
+; %if.then.3 2^-63 ~ 1.08e-19 (clamped up from 2^-93)
+; %latch 1 = 1
+;
+; BFI-LABEL: block-frequency-info: nested_blocks_almost_never_entered
+; BFI-NEXT: - entry: float = 1.0,
+; BFI-NEXT: - loop: float = 1000.0,
+; BFI-NEXT: - if.then.1: float = 0.00000046566,
+; BFI-NEXT: - if.then.2: float = 0.00000000000000021684,
+; BFI-NEXT: - if.then.3: float = 0.00000000000000005421,
+; BFI-NEXT: - latch: float = 1000.0,
+; BFI-NEXT: - exit: float = 1.0,
+;
+; VPLAN-LABEL: VPlan for loop in 'nested_blocks_almost_never_entered'
+; VPLAN: if.then.1:
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp sgt ir<%l>, ir<1>, ir<%c.0> (!vplan.execution.frequency 4294967296 (4.657e-08%))
+; VPLAN-NEXT: Successor(s): if.then.2
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then.2:
+; VPLAN-NEXT: EMIT vp<[[AND:%.+]]> = logical-and ir<%c.0>, ir<%c.1>
+; VPLAN-NEXT: EMIT ir<%c.2> = icmp sgt ir<%l>, ir<2>, vp<[[AND]]> (!vplan.execution.frequency 2 (2.168e-17%))
+; VPLAN-NEXT: Successor(s): if.then.3
+; VPLAN-EMPTY:
+; VPLAN-NEXT: if.then.3:
+; VPLAN-NEXT: EMIT vp<[[MASK:%.+]]> = logical-and vp<[[AND]]>, ir<%c.2>
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>, vp<[[MASK]]> (!vplan.execution.frequency 1 (1.084e-17%))
+; VPLAN-NEXT: Successor(s): latch
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.idx = getelementptr inbounds i32, ptr %idx, i64 %iv
+ %l = load i32, ptr %gep.idx, align 4
+ %c.0 = icmp sgt i32 %l, 0
+ br i1 %c.0, label %if.then.1, label %latch, !prof !7
+
+if.then.1:
+ %c.1 = icmp sgt i32 %l, 1
+ br i1 %c.1, label %if.then.2, label %latch, !prof !7
+
+if.then.2:
+ %c.2 = icmp sgt i32 %l, 2
+ br i1 %c.2, label %if.then.3, label %latch, !prof !7
+
+if.then.3:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 1, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
!0 = !{!"branch_weights", i32 1, i32 3}
!1 = !{!"branch_weights", i32 1, i32 2}
!2 = !{!"branch_weights", i32 1, i32 1}
@@ -782,3 +973,4 @@ exit:
!7 = !{!"branch_weights", i32 1, i32 2147483647}
!8 = !{!"branch_weights", i32 1, i32 1, i32 1, i32 1, i32 1}
!9 = !{!"branch_weights", i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1}
+!12 = !{!"branch_weights", i32 2, i32 2863311531, i32 2863311531, i32 2863311531}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 1bd959b67b672..1afd9a1955028 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -30,7 +30,6 @@
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeScalarizationDecisions
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeCallWideningDecisions
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dropUnguardedExecutionProbabilities
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeFindIVReductions
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
index cb8fe6f49d926..e89172c1286bb 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
@@ -12,7 +12,7 @@
; RUN: -vplan-print-after=dissolveLoopRegions -disable-output %s 2>&1 \
; RUN: | FileCheck --strict-whitespace --check-prefix=DISSOLVE %s
-; Track the execution probability of a predicated block through VPlan, from the
+; Track the execution frequency of a predicated block through VPlan, from the
; branch weights of the original loop to the branch weights of the branch
; guarding the predicated block.
@@ -41,11 +41,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; PREDICATE-NEXT: Successor(s): if.then
; PREDICATE-EMPTY:
; PREDICATE-NEXT: if.then:
-; PREDICATE-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25%)
+; PREDICATE-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.frequency 2305843009213693952 (25%))
; PREDICATE-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; PREDICATE-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; PREDICATE-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; PREDICATE-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25%)
+; PREDICATE-NEXT: EMIT store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.frequency 2305843009213693952 (25%))
; PREDICATE-NEXT: Successor(s): latch
; PREDICATE-EMPTY:
; PREDICATE-NEXT: latch:
@@ -84,11 +84,11 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; CONSTRUCT-NEXT: Successor(s): if.then
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: if.then:
-; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.probability 25%)
+; CONSTRUCT-NEXT: EMIT ir<%add> = add ir<%i>, ir<1>, ir<%cmp> (!vplan.execution.frequency 2305843009213693952 (25%))
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%t> = trunc ir<%add> to i16
; CONSTRUCT-NEXT: EMIT-SCALAR ir<%ext> = sext ir<%t> to i64
; CONSTRUCT-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%ext>
-; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.probability 25%)
+; CONSTRUCT-NEXT: REPLICATE store ir<%add>, ir<%gep.a>, ir<%cmp> (!vplan.execution.frequency 2305843009213693952 (25%))
; CONSTRUCT-NEXT: Successor(s): latch
; CONSTRUCT-EMPTY:
; CONSTRUCT-NEXT: latch:
@@ -131,7 +131,7 @@ define void @predicated_block(ptr noalias %a, ptr noalias %idx) {
; REGION-EMPTY:
; REGION-NEXT: <xVFxUF> pred.store: {
; REGION-NEXT: pred.store.entry:
-; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!vplan.execution.probability 25%)
+; REGION-NEXT: BRANCH-ON-MASK ir<%cmp> (!vplan.execution.frequency 2305843009213693952 (25%))
; REGION-NEXT: Successor(s): pred.store.if, pred.store.continue
; REGION-EMPTY:
; REGION-NEXT: pred.store.if:
More information about the llvm-commits
mailing list