[llvm] [VPlan] Cost scalar IV steps in replicate regions. (PR #221995)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 06:15:48 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/221995
Replace the return 0 bail-out in VPScalarIVStepsRecipe::computeCost in
replicate regions by properly scaling by the execution probability of
the region.
VPlan-based estimates of execution probabilities depend on
https://github.com/llvm/llvm-project/pull/216172 (include in PR)
Relevant commit is https://github.com/llvm/llvm-project/commit/b13864475308bb69ca50604d8056d3045c1f2f0e
>From 000ec49289b2f2d05927f6cac18d8cfd7a9fdb49 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 29 Jul 2026 09:28:31 +0100
Subject: [PATCH 1/2] [VPlan] Record estimated branch probabilities on VPlan0
for cost modeling.
Update execution frequency/probability tracking in VPlan to also include
estimates from BFI, and use the information in VPlan for VPlan-based
cost computations, removing another remaining IR-based lookup.
We need to differentiate between estimated and concrete profile
information when annotating branches with information from BFI (using
custom !vplan.prof.estimated metadata instead of !prof) and track if a
computed frequency contains any estimates using an extra bool which is
also encoded in the metadata (so we do not synthesize !prof
for such frequencies during codegen).
The new code should only request BFI if needed, and compile-time impact
should be in the noise:
https://llvm-compile-time-tracker.com/compare.php?from=8436dd4a2bfe57d0b3fb6c9c11c69d75c1d1bccb&to=4d60694fed5e5765d4f5a2a72aac7b8674e512e2&stat=instructions:u
---
.../llvm/Transforms/Vectorize/LoopVectorize.h | 2 +
.../Vectorize/LoopVectorizationPlanner.h | 7 ++-
.../Transforms/Vectorize/LoopVectorize.cpp | 51 +++++++++++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 43 +++++++++++++-
.../Vectorize/VPlanConstruction.cpp | 50 +++++++++++++---
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 6 +-
.../Transforms/Vectorize/VPlanPredicator.cpp | 6 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 58 +++++++++++--------
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 +++-
.../Transforms/Vectorize/VPlanTransforms.h | 4 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 9 +--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 26 +++++----
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 5 +-
.../VPlan/execution-frequencies-match-bfi.ll | 11 ++--
.../replicate-region-cost-probability.ll | 9 +--
15 files changed, 209 insertions(+), 88 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index 5eba1a0f8527c..73c607ad1cf5a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -66,6 +66,7 @@ namespace llvm {
class AssumptionCache;
class BlockFrequencyInfo;
+class BranchProbabilityInfo;
class DemandedBits;
class DominatorTree;
class Function;
@@ -146,6 +147,7 @@ struct LoopVectorizePass : public OptionalPassInfoMixin<LoopVectorizePass> {
TargetTransformInfo *TTI;
DominatorTree *DT;
std::function<BlockFrequencyInfo &()> GetBFI;
+ std::function<const BranchProbabilityInfo &()> GetBPI;
TargetLibraryInfo *TLI;
DemandedBits *DB;
AssumptionCache *AC;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 972f8f8e642e2..6b9a2effd6422 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -36,6 +36,7 @@ class GeneratedRTChecks;
namespace llvm {
+class BranchProbabilityInfo;
class LoopInfo;
class DominatorTree;
class LoopVectorizationLegality;
@@ -891,6 +892,9 @@ class LoopVectorizationPlanner {
OptimizationRemarkEmitter *ORE;
+ /// Lazily fetch BranchProbabilityInfo, independent of BlockFrequencyInfo.
+ std::function<const BranchProbabilityInfo &()> GetBPI;
+
SmallVector<VPlanPtr, 4> VPlans;
/// Profitable vector factors.
@@ -921,7 +925,8 @@ class LoopVectorizationPlanner {
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
std::unique_ptr<LoopVectorizationCostModel> CM,
VFSelectionContext &Config, InterleavedAccessInfo &IAI,
- PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
+ PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE,
+ std::function<const BranchProbabilityInfo &()> GetBPI);
~LoopVectorizationPlanner();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e027dfa834759..661e1f0957170 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -80,6 +80,7 @@
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/CycleAnalysis.h"
@@ -5347,8 +5348,19 @@ void VPCostContext::invalidateWideningDecision(Instruction *I,
LoopVectorizationCostModel::CM_InvalidatedDecision, 0);
}
-uint64_t VPCostContext::getPredBlockCostDivisor(BasicBlock *BB) const {
- return CM.getPredBlockCostDivisor(CostKind, BB);
+uint64_t
+VPCostContext::getPredBlockCostDivisor(const VPRegionBlock *Region) const {
+ if (CostKind == TTI::TCK_CodeSize)
+ return 1;
+ std::optional<VPExecutionFrequency> Freq =
+ Region->getEntryBranchOnMask()->getExecutionFrequency();
+ if (!Freq)
+ return 1;
+ // A recorded frequency is neither zero nor always-executing, so the
+ // probability is non-zero and the division below is safe.
+ return divideNearest(
+ BranchProbability::getDenominator(),
+ vputils::getExecutionProbability(Freq->Freq).getNumerator());
}
bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
@@ -5669,9 +5681,11 @@ LoopVectorizationPlanner::LoopVectorizationPlanner(
const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
- OptimizationRemarkEmitter *ORE)
+ OptimizationRemarkEmitter *ORE,
+ std::function<const BranchProbabilityInfo &()> GetBPI)
: OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
+ CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE),
+ GetBPI(GetBPI) {}
LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
@@ -6331,16 +6345,16 @@ static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
Edges += VPBB->getNumSuccessors();
uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
- DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies =
- vputils::computeExecutionFrequencies(Blocks);
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies = vputils::computeExecutionFrequencies(Blocks);
for (const auto &[VPBB, BB] :
zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
// Compare at BranchProbability's coarser resolution, which is as precise as
// BFI's frequencies get.
- std::optional<BlockFrequency> Freq = Frequencies.lookup(VPBB);
+ std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
if (!Freq)
continue;
- BranchProbability Computed = vputils::getExecutionProbability(*Freq);
+ BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
// Clamp to the header's frequency, which BFI's rounding may exceed.
uint64_t BBFreq = BFI.getBlockFreq(BB).getFrequency();
@@ -6379,9 +6393,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// Create initial base VPlan0, to serve as common starting point for all
// candidates built later for specific VF ranges.
- auto VPlan0 = VPlanTransforms::buildVPlan0(OrigLoop, *LI,
- Legal->getWidestInductionType(),
- PSE, LVer ? &*LVer : nullptr);
+ auto VPlan0 = VPlanTransforms::buildVPlan0(
+ OrigLoop, *LI, Legal->getWidestInductionType(), PSE,
+ LVer ? &*LVer : nullptr, GetBPI);
VPDominatorTree VPDT(*VPlan0);
if (const LoopAccessInfo *LAI = Legal->getLAI())
@@ -7930,7 +7944,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
L, LI, DT, TLI, *TTI, &LVL,
std::make_unique<LoopVectorizationCostModel>(
SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
- Config, IAI, PSE, ORE);
+ Config, IAI, PSE, ORE, GetBPI);
EpilogueLowering EpilogueTailLoweringStatus =
getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
@@ -8340,12 +8354,21 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
- GetBFI = [this, &AM, &F]() -> BlockFrequencyInfo & {
- // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
+ // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
+ // Both BlockFrequencyAnalysis and BranchProbabilityAnalysis depend on it, so
+ // drop the stale result before either is (re-)computed.
+ auto ClearStaleCycleInfo = [this, &AM, &F] {
if (CFGChanged && AM.getCachedResult<CycleAnalysis>(F))
AM.clearAnalysis<CycleAnalysis>(F);
+ };
+ GetBFI = [&AM, &F, ClearStaleCycleInfo]() -> BlockFrequencyInfo & {
+ ClearStaleCycleInfo();
return AM.getResult<BlockFrequencyAnalysis>(F);
};
+ GetBPI = [&AM, &F, ClearStaleCycleInfo]() -> const BranchProbabilityInfo & {
+ ClearStaleCycleInfo();
+ return AM.getResult<BranchProbabilityAnalysis>(F);
+ };
LoopVectorizeResult Result = runImpl(F);
if (!Result.MadeAnyChange)
return PreservedAnalyses::all();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index e9f18506af7c2..af7ebccb713ef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1177,6 +1177,14 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
VPCostContext &Ctx) const;
};
+/// The frequency with which a recipe executes, relative to the entry of the
+/// loop region. IsEstimated is set if any branch weight it was composed from
+/// was estimated from static heuristics.
+struct VPExecutionFrequency {
+ BlockFrequency Freq;
+ bool IsEstimated;
+};
+
/// Helper to manage IR metadata for recipes. It filters out metadata that
/// cannot be propagated.
class LLVM_ABI_FOR_TEST VPIRMetadata {
@@ -1186,6 +1194,10 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
static constexpr StringLiteral ExecutionFrequencyMDName =
"vplan.execution.frequency";
+ /// Name of the VPlan-internal metadata kind holding estimated branch weights.
+ static constexpr StringLiteral EstimatedProfileMDName =
+ "vplan.prof.estimated";
+
/// Returns the ID of the metadata kind named \p Kind, taking the context from
/// any attached node; all belong to the context of the VPlan's function.
unsigned getMDKindID(StringRef Kind) const {
@@ -1193,6 +1205,12 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
return Metadata.front().second->getContext().getMDKindID(Kind);
}
+ /// Returns the node attached under the VPlan-internal metadata kind named
+ /// \p Kind, or nullptr if there is none.
+ MDNode *getInternalMetadata(StringRef Kind) const {
+ return Metadata.empty() ? nullptr : getMetadata(getMDKindID(Kind));
+ }
+
public:
VPIRMetadata() = default;
@@ -1240,16 +1258,35 @@ class LLVM_ABI_FOR_TEST VPIRMetadata {
}
/// Record that the recipe executes with frequency \p Freq, relative to the
- /// entry of the loop region; see vputils::AlwaysExecutesFreq.
- void setExecutionFrequency(std::optional<BlockFrequency> Freq,
+ /// entry of the loop region.
+ void setExecutionFrequency(std::optional<VPExecutionFrequency> Freq,
LLVMContext &Ctx);
/// Returns the frequency recorded by setExecutionFrequency, if any.
- std::optional<BlockFrequency> getExecutionFrequency() const;
+ std::optional<VPExecutionFrequency> getExecutionFrequency() const;
/// Drop the frequency recorded by setExecutionFrequency, if any.
void clearExecutionFrequency();
+ /// Returns the branch weights recorded for this terminator, preferring real
+ /// profile data over an estimate, or nullptr if there are none.
+ MDNode *getBranchWeights() const {
+ MDNode *Node = getMetadata(LLVMContext::MD_prof);
+ return Node ? Node : getInternalMetadata(EstimatedProfileMDName);
+ }
+
+ /// Returns true if the weights returned by getBranchWeights are estimated.
+ bool hasEstimatedBranchWeights() const {
+ return getInternalMetadata(EstimatedProfileMDName);
+ }
+
+ /// Set estimated branch weights to \p Node.
+ void setEstimatedBranchWeights(MDNode *Node) {
+ assert(!getMetadata(LLVMContext::MD_prof) &&
+ "real profile data takes precedence over an estimate");
+ setMetadata(Node->getContext().getMDKindID(EstimatedProfileMDName), Node);
+ }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print metadata with node IDs.
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a2006ebf4bb4f..384dc5bc85d4a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -20,7 +20,9 @@
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
+#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVectorExtras.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
@@ -53,6 +55,12 @@ class PlainCFGBuilder {
// Loop versioning for alias metadata.
LoopVersioning *LVer;
+ // Lazily provides branch probabilities for the incoming IR.
+ function_ref<const BranchProbabilityInfo &()> GetBPI;
+
+ // The BranchProbabilityInfo returned by GetBPI, cached on first use.
+ const BranchProbabilityInfo *BPI = nullptr;
+
// Vectorization plan that we are working on.
std::unique_ptr<VPlan> Plan;
@@ -79,10 +87,12 @@ class PlainCFGBuilder {
#endif
VPValue *getOrCreateVPOperand(Value *IRVal);
void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
+ VPIRMetadata getTerminatorMetadata(Instruction &Term);
public:
- PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer, Type *IdxTy)
- : TheLoop(Lp), LI(LI), LVer(LVer),
+ PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer, Type *IdxTy,
+ function_ref<const BranchProbabilityInfo &()> GetBPI)
+ : TheLoop(Lp), LI(LI), LVer(LVer), GetBPI(GetBPI),
Plan(std::make_unique<VPlan>(Lp, IdxTy)) {}
/// Build plain CFG for TheLoop and connect it to Plan's entry.
@@ -183,6 +193,26 @@ VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
return NewVPVal;
}
+// Returns the metadata to preserve for terminator \p Term.
+VPIRMetadata PlainCFGBuilder::getTerminatorMetadata(Instruction &Term) {
+ VPIRMetadata MD(Term);
+ if (!GetBPI || MD.getMetadata(LLVMContext::MD_prof))
+ return MD;
+ // Estimates are only read for edges inside the loop region.
+ if (!TheLoop->isInnermost() || Term.getParent() == TheLoop->getLoopLatch())
+ return MD;
+ // The weights describe the edges leaving Term in the order of its successors,
+ // matching the successor order of the VPBasicBlock created for Term's parent.
+ if (!BPI)
+ BPI = &GetBPI();
+ auto Weights = map_to_vector(seq(Term.getNumSuccessors()), [&](unsigned I) {
+ return BPI->getEdgeProbability(Term.getParent(), I).getNumerator();
+ });
+ MD.setEstimatedBranchWeights(
+ MDBuilder(Plan->getContext()).createBranchWeights(Weights));
+ return MD;
+}
+
// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
// counterpart. This function must be invoked in RPO so that the operands of a
// VPInstruction in \p BB have been visited before (except for Phi nodes).
@@ -207,7 +237,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// recipes.
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, {},
- VPIRMetadata(*Inst), Inst->getDebugLoc());
+ getTerminatorMetadata(*Inst),
+ Inst->getDebugLoc());
continue;
}
@@ -219,7 +250,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
for (auto Case : SI->cases())
Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, {},
- VPIRMetadata(*Inst), Inst->getDebugLoc());
+ getTerminatorMetadata(*Inst),
+ Inst->getDebugLoc());
continue;
}
@@ -598,11 +630,11 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
/// To make RUN_VPLAN_PASS print initial VPlan.
static void printAfterInitialConstruction(VPlan &) {}
-std::unique_ptr<VPlan>
-VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
- PredicatedScalarEvolution &PSE,
- LoopVersioning *LVer) {
- PlainCFGBuilder Builder(TheLoop, &LI, LVer, InductionTy);
+std::unique_ptr<VPlan> VPlanTransforms::buildVPlan0(
+ Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
+ PredicatedScalarEvolution &PSE, LoopVersioning *LVer,
+ function_ref<const BranchProbabilityInfo &()> GetBPI) {
+ PlainCFGBuilder Builder(TheLoop, &LI, LVer, InductionTy, GetBPI);
std::unique_ptr<VPlan> VPlan0 = Builder.buildPlainCFG();
addInitialSkeleton(*VPlan0, InductionTy, PSE, TheLoop);
simplifyLiveInsWithSCEV(*VPlan0, PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 2838a9e4895a6..a4ae2d57548a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -356,9 +356,9 @@ struct VPCostContext {
/// transform replaced the original recipe.
void invalidateWideningDecision(Instruction *I, ElementCount VF);
- /// \returns how much the cost of a predicated block should be divided by.
- /// Forwards to LoopVectorizationCostModel::getPredBlockCostDivisor.
- uint64_t getPredBlockCostDivisor(BasicBlock *BB) const;
+ /// \returns how much the cost of the block predicated by replicate region
+ /// \p Region should be divided by.
+ uint64_t getPredBlockCostDivisor(const VPRegionBlock *Region) const;
/// Returns true if \p I is known to be scalarized at \p VF.
bool willBeScalarized(Instruction *I, ElementCount VF) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index cbc41e82fa87c..e056b2b613b33 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -406,8 +406,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
Header);
// Non-outer regions with VPBBs only are supported at the moment.
auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
- DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies =
- vputils::computeExecutionFrequencies(Blocks);
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies = vputils::computeExecutionFrequencies(Blocks);
VPPredicator Predicator(Plan);
for (VPBasicBlock *VPBB : Blocks) {
@@ -423,7 +423,7 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
// Mask all VPInstructions in the block and record the frequency with
// which the masked recipes execute.
- std::optional<BlockFrequency> Freq = Frequencies.lookup(VPBB);
+ std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ff212663a3fac..ec150e5150d4c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2128,41 +2128,43 @@ void VPIRPhi::printRecipe(raw_ostream &O, const Twine &Indent,
void VPIRMetadata::applyMetadata(Instruction &I) const {
if (Metadata.empty())
return;
- // The execution frequency is VPlan-internal and must not reach IR.
+ // Frequencies and estimated branch weights are VPlan-internal and must not
+ // reach IR.
unsigned ExecFreqKind = getMDKindID(ExecutionFrequencyMDName);
+ unsigned EstProfKind = getMDKindID(EstimatedProfileMDName);
for (const auto &[Kind, Node] : Metadata)
- if (Kind != ExecFreqKind)
+ if (Kind != ExecFreqKind && Kind != EstProfKind)
I.setMetadata(Kind, Node);
}
/// Returns the execution frequency recorded in \p Node.
-static BlockFrequency getExecutionFrequencyFromMD(const MDNode *Node) {
+static VPExecutionFrequency getExecutionFrequencyFromMD(const MDNode *Node) {
+ assert(Node->getNumOperands() <= 2 && "unexpected frequency node shape");
uint64_t Freq =
mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue();
assert(Freq <= vputils::AlwaysExecutesFreq &&
"frequency cannot exceed the one of an always executing block");
- return BlockFrequency(Freq);
+ return {BlockFrequency(Freq), Node->getNumOperands() == 2};
}
-void VPIRMetadata::setExecutionFrequency(std::optional<BlockFrequency> Freq,
- LLVMContext &Ctx) {
+void VPIRMetadata::setExecutionFrequency(
+ std::optional<VPExecutionFrequency> Freq, LLVMContext &Ctx) {
// A recipe that never or always executes needs no annotation.
- if (!Freq || Freq->getFrequency() == 0 ||
- Freq->getFrequency() == vputils::AlwaysExecutesFreq)
+ if (!Freq || Freq->Freq.getFrequency() == 0 ||
+ Freq->Freq.getFrequency() == vputils::AlwaysExecutesFreq)
return;
- Constant *Frequency =
- ConstantInt::get(Type::getInt64Ty(Ctx), Freq->getFrequency());
- setMetadata(Ctx.getMDKindID(ExecutionFrequencyMDName),
- MDNode::get(Ctx, {ConstantAsMetadata::get(Frequency)}));
+ SmallVector<llvm::Metadata *, 2> Ops = {ConstantAsMetadata::get(
+ ConstantInt::get(Type::getInt64Ty(Ctx), Freq->Freq.getFrequency()))};
+ if (Freq->IsEstimated)
+ Ops.push_back(ConstantAsMetadata::get(ConstantInt::getTrue(Ctx)));
+ setMetadata(Ctx.getMDKindID(ExecutionFrequencyMDName), MDNode::get(Ctx, Ops));
}
-std::optional<BlockFrequency> VPIRMetadata::getExecutionFrequency() const {
- if (Metadata.empty())
- return std::nullopt;
- MDNode *Node = getMetadata(getMDKindID(ExecutionFrequencyMDName));
- if (!Node)
- return std::nullopt;
- return getExecutionFrequencyFromMD(Node);
+std::optional<VPExecutionFrequency>
+VPIRMetadata::getExecutionFrequency() const {
+ if (MDNode *Node = getInternalMetadata(ExecutionFrequencyMDName))
+ return getExecutionFrequencyFromMD(Node);
+ return std::nullopt;
}
void VPIRMetadata::clearExecutionFrequency() {
@@ -2201,15 +2203,21 @@ void VPIRMetadata::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
// Print the values of branch weights, which are more informative than the
// ID of the metadata node holding them.
SmallVector<uint32_t> Weights;
- if (Kind == LLVMContext::MD_prof && extractBranchWeights(Node, Weights)) {
+ bool IsEstimatedProfile = MDNames[Kind] == EstimatedProfileMDName;
+ if ((Kind == LLVMContext::MD_prof || IsEstimatedProfile) &&
+ extractBranchWeights(Node, Weights)) {
+ if (IsEstimatedProfile)
+ O << "estimated ";
O << "{";
interleaveComma(Weights, O);
O << "}";
} else if (MDNames[Kind] == ExecutionFrequencyMDName) {
// Print the frequency together with the probability it corresponds to.
- uint64_t Freq = getExecutionFrequencyFromMD(Node).getFrequency();
- O << Freq
- << format(" (%.4g%%)", 100.0 * Freq / vputils::AlwaysExecutesFreq);
+ auto [Freq, IsEstimated] = getExecutionFrequencyFromMD(Node);
+ O << Freq.getFrequency()
+ << format(" (%.4g%%%s)",
+ 100.0 * Freq.getFrequency() / vputils::AlwaysExecutesFreq,
+ IsEstimated ? ", estimated" : "");
} else {
Node->printAsOperand(O, M);
}
@@ -4049,7 +4057,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
// Scale the cost by the probability of executing the predicated blocks.
// This assumes the predicated block for each vector lane is equally
// likely.
- ScalarCost /= Ctx.getPredBlockCostDivisor(UI->getParent());
+ ScalarCost /= Ctx.getPredBlockCostDivisor(getRegion());
return ScalarCost;
}
case Instruction::Load:
@@ -4108,7 +4116,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
if (ParentRegion && ParentRegion->isReplicator()) {
if (!PtrSCEV)
break;
- Cost /= Ctx.getPredBlockCostDivisor(UI->getParent());
+ Cost /= Ctx.getPredBlockCostDivisor(ParentRegion);
Cost += Ctx.TTI.getCFInstrCost(Instruction::CondBr, Ctx.CostKind);
auto *VecI1Ty = VectorType::get(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 63011d01ad9ce..44eb6bea9be26 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -496,12 +496,16 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
// If only one of the two is known, the higher one is unknown, so the
// result must be unknown too.
VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
- std::optional<BlockFrequency> Freq1 =
+ std::optional<VPExecutionFrequency> Freq1 =
Region1->getEntryBranchOnMask()->getExecutionFrequency();
- std::optional<BlockFrequency> Freq2 = Guard2->getExecutionFrequency();
+ std::optional<VPExecutionFrequency> Freq2 = Guard2->getExecutionFrequency();
if (Freq1 && Freq2) {
- if (*Freq2 < *Freq1)
+ if (Freq2->Freq < Freq1->Freq) {
+ // Freq1's frequency is taken, but it is only as trustworthy as the
+ // less trustworthy of the two.
+ Freq1->IsEstimated |= Freq2->IsEstimated;
Guard2->setExecutionFrequency(Freq1, Plan.getContext());
+ }
} else if (Freq2) {
Guard2->clearExecutionFrequency();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index fef0c14415091..ea5a647cb52a6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -24,6 +24,7 @@
namespace llvm {
+class BranchProbabilityInfo;
class InductionDescriptor;
class Instruction;
class Loop;
@@ -156,7 +157,8 @@ struct VPlanTransforms {
/// >[ ] <-- original loop exit block(s), wrapped in VPIRBasicBlocks.
LLVM_ABI_FOR_TEST static std::unique_ptr<VPlan>
buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
- PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr);
+ PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr,
+ function_ref<const BranchProbabilityInfo &()> GetBPI = nullptr);
/// Replace VPPhi recipes in \p Plan's header with corresponding
/// VPHeaderPHIRecipe subclasses for inductions, reductions, and
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 0524f9c08be5b..d69eff676e5de 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -676,13 +676,14 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
}
/// Converts the frequency \p Freq with which a block is entered to branch
-/// weights for the branch guarding it, or nullptr if \p Freq is unknown.
+/// weights for the branch guarding it, or nullptr if \p Freq is unknown or
+/// estimated.
static MDNode *
-convertFrequencyToBranchWeights(std::optional<BlockFrequency> Freq,
+convertFrequencyToBranchWeights(std::optional<VPExecutionFrequency> Freq,
LLVMContext &Ctx) {
- if (!Freq)
+ if (!Freq || Freq->IsEstimated)
return nullptr;
- BranchProbability P = vputils::getExecutionProbability(*Freq);
+ BranchProbability P = vputils::getExecutionProbability(Freq->Freq);
// Use the numerators of P and its complement as weights and reduce them via
// gcd to keep them small. Neither is zero, as P is neither zero nor one.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index befc2d477aac4..a696bb36a5132 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1192,8 +1192,7 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
// below still visits each of them.
SmallVector<uint32_t> Weights;
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
- if (!Term ||
- !extractBranchWeights(Term->getMetadata(LLVMContext::MD_prof), Weights) ||
+ if (!Term || !extractBranchWeights(Term->getBranchWeights(), Weights) ||
Weights.size() != Successors.size())
Weights.assign(Successors.size(), 0);
uint64_t Total = sum_of(Weights, uint64_t(0));
@@ -1223,31 +1222,36 @@ static BlockFrequency scaleKeepingNonZero(BlockFrequency Freq,
return Scaled;
}
-DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>>
+DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
assert(!Blocks.empty() && "expected at least the header block");
// Push each block's frequency along its outgoing edges. Blocks is a DAG in
// reverse post-order (the loop region's backedge is implicit), so a block's
// frequency is final by the time it is visited.
- DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>> Frequencies;
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies;
Frequencies.reserve(Blocks.size());
// The header (first block) always executes, the others start out unreachable.
- Frequencies[Blocks.front()] = BlockFrequency(AlwaysExecutesFreq);
+ Frequencies[Blocks.front()] = {BlockFrequency(AlwaysExecutesFreq), false};
for (VPBasicBlock *VPBB : Blocks.drop_front())
- Frequencies[VPBB] = BlockFrequency();
+ Frequencies[VPBB] = {BlockFrequency(), false};
for (VPBasicBlock *VPBB : Blocks) {
- std::optional<BlockFrequency> SrcFreq = Frequencies.at(VPBB);
+ std::optional<VPExecutionFrequency> Src = Frequencies.at(VPBB);
+ auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
+ bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- std::optional<BlockFrequency> &SuccFreq = Frequencies.at(Succ);
+ std::optional<VPExecutionFrequency> &SuccFreq = Frequencies.at(Succ);
// An unknown edge or predecessor poisons the successor.
- if (!SrcFreq || EdgeProb.isUnknown() || !SuccFreq) {
+ if (!Src || EdgeProb.isUnknown() || !SuccFreq) {
SuccFreq = std::nullopt;
continue;
}
// The sum can only exceed AlwaysExecutesFreq by rounding.
- SuccFreq = std::min(BlockFrequency(AlwaysExecutesFreq),
- *SuccFreq + scaleKeepingNonZero(*SrcFreq, EdgeProb));
+ SuccFreq->Freq =
+ std::min(BlockFrequency(AlwaysExecutesFreq),
+ SuccFreq->Freq + scaleKeepingNonZero(Src->Freq, EdgeProb));
+ SuccFreq->IsEstimated |= Src->IsEstimated || TermIsEstimated;
}
}
return Frequencies;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 4bb03cb6b691b..828a2f5ab472d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -232,10 +232,11 @@ inline constexpr uint64_t AlwaysExecutesFreq = 1ULL << 63;
BranchProbability getExecutionProbability(BlockFrequency Freq);
/// Computes for each block in \p Blocks, which must be in reverse post-order,
-/// the frequency with which it executes relative to the first (header) block.
+/// the frequency with which it executes relative to the first (header) block,
+/// and whether that frequency was composed using any estimated branch weights.
/// The frequency of a block is the sum over its incoming edges, or std::nullopt
/// if any edge on a path reaching it lacks branch weights.
-DenseMap<const VPBasicBlock *, std::optional<BlockFrequency>>
+DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks);
namespace detail {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
index 0b0a698b42bf2..244c9add3b2dd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
@@ -223,14 +223,15 @@ exit:
}
define void @second_branch_without_weights(ptr noalias %a, ptr noalias %b, ptr noalias %idx) {
-; %merge's branch has no weights, so everything it reaches is unknown and stays
-; unannotated. %merge itself is still known, as both of its incoming edges are.
+; %merge's branch has no weights, so BranchProbabilityInfo estimates them.
+; %if.then.2's frequency is composed through that estimated edge, so it is
+; marked estimated too.
;
; %loop 1000/1000 = 1
; %if.then.1 250/1000 = 1/4
; %merge 1000/1000 = 1
-; %if.then.2 unknown
-; %latch unknown
+; %if.then.2 625/1000 = 5/8 (estimated)
+; %latch 1000/1000 = 1
;
; BFI-LABEL: block-frequency-info: second_branch_without_weights
; BFI-NEXT: - entry: float = 1.0,
@@ -252,7 +253,7 @@ define void @second_branch_without_weights(ptr noalias %a, ptr noalias %b, ptr n
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then.2:
; VPLAN-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0>{{$}}
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.b>, ir<%c.0> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; VPLAN-NEXT: Successor(s): latch
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
index 1928c2c3e995a..8b4477346b55b 100644
--- a/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
@@ -72,8 +72,9 @@ exit:
}
; Without profile data the cost model is still self-contained: the recorded
-; reciprocal comes from block frequency info, which defaults to an even 50/50
-; split for the unprofiled predicate (reciprocal 2), so cost 10 / 2 -> 5.
+; reciprocal comes from branch weights estimated by BranchProbabilityInfo. For
+; this comparison against zero that heuristic favors the predicate at 5/8
+; (round(1/0.625) = 2), so cost 10 / 2 -> 5.
define void @predicated_udiv_no_profile(ptr %a, i32 %n) {
; CHECK-LABEL: 'predicated_udiv_no_profile'
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
@@ -113,12 +114,12 @@ define void @merged_regions_use_conservative_probability(ptr noalias %a, ptr noa
; CHECK-LABEL: 'merged_regions_use_conservative_probability'
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7:%[0-9]+]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6:%[0-9]+]]>, ir<%gep>.1
-; CHECK: Cost of 0 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6]]>, ir<%gep>.1
-; CHECK: Cost of 0 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
; CHECK: Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
;
>From b13864475308bb69ca50604d8056d3045c1f2f0e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 5 Sep 2026 19:36:38 +0100
Subject: [PATCH 2/2] [VPlan] Cost scalar IV steps in replicate regions.
Replace the return 0 bail-out in VPScalarIVStepsRecipe::computeCost in
replicate regions by properly scaling by the execution probability of
the region.
VPlan-based estimates of execution probabilities depend on
https://github.com/llvm/llvm-project/pull/216172 (include in PR)
---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 +-
.../AArch64/induction-costs-sve.ll | 48 +-----
.../AArch64/scalar-steps-cost.ll | 4 +-
.../LoopVectorize/ARM/optsize_minsize.ll | 143 ++----------------
.../X86/CostModel/store-scalarization-cost.ll | 4 +-
5 files changed, 30 insertions(+), 185 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ec150e5150d4c..9a0b7dddfb963 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3182,11 +3182,6 @@ InstructionCost VPScalarIVStepsRecipe::computeCost(ElementCount VF,
if (!BaseIVTy->isIntegerTy())
return 0;
- // TODO: Add support for predicated regions. Requires scaling the cost by the
- // probability of entering the block.
- if (getRegion() && getRegion()->isReplicator())
- return 0;
-
// If only the first lane is used, then there won't be any code that remains
// in the loop for the first unrolled part.
if (vputils::onlyFirstLaneUsed(this))
@@ -3211,8 +3206,15 @@ InstructionCost VPScalarIVStepsRecipe::computeCost(ElementCount VF,
// %gep2 = getelementptr i8, ptr %base_gep, i32 1
// Therefore, in reality the cost is somewhere betwen 1*AddCost and
// (NumLanes - 1) * AddCost. For now, assume the cost of a single add.
- return Ctx.TTI.getArithmeticInstrCost(Instruction::Add, BaseIVTy,
- Ctx.CostKind);
+ //
+ // If the steps are generated inside a replicate region, scale by execution
+ // probability.
+ InstructionCost Cost =
+ Ctx.TTI.getArithmeticInstrCost(Instruction::Add, BaseIVTy, Ctx.CostKind);
+ const VPRegionBlock *Region = getRegion();
+ if (Region && Region->isReplicator())
+ Cost /= Ctx.getPredBlockCostDivisor(Region);
+ return Cost;
}
void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 7c3f140edf859..a99bd18aea6a9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -696,51 +696,7 @@ define void @exit_cond_zext_iv_store64(ptr %dst, i64 %N) {
;
; PRED-LABEL: define void @exit_cond_zext_iv_store64(
; PRED-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; PRED-NEXT: [[ENTRY:.*:]]
-; PRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; PRED-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; PRED: [[VECTOR_SCEVCHECK]]:
-; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 1)
-; PRED-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
-; PRED-NEXT: [[TMP3:%.*]] = add i32 1, [[TMP2]]
-; PRED-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 1
-; PRED-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
-; PRED-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
-; PRED-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; PRED: [[VECTOR_PH]]:
-; PRED-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 1
-; PRED-NEXT: [[TMP7:%.*]] = and i64 [[N_RND_UP]], 1
-; PRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP7]]
-; PRED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
-; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
-; PRED: [[VECTOR_BODY]]:
-; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; PRED-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
-; PRED-NEXT: [[TMP8:%.*]] = icmp ule <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PRED-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP8]], i64 0
-; PRED-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; PRED: [[PRED_STORE_IF]]:
-; PRED-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
-; PRED-NEXT: store i64 0, ptr [[TMP10]], align 8
-; PRED-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; PRED: [[PRED_STORE_CONTINUE]]:
-; PRED-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i64 1
-; PRED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
-; PRED: [[PRED_STORE_IF1]]:
-; PRED-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
-; PRED-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP12]]
-; PRED-NEXT: store i64 0, ptr [[TMP13]], align 8
-; PRED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; PRED: [[PRED_STORE_CONTINUE2]]:
-; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; PRED-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
-; PRED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; PRED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; PRED: [[MIDDLE_BLOCK]]:
-; PRED-NEXT: br label %[[EXIT:.*]]
-; PRED: [[SCALAR_PH]]:
+; PRED-NEXT: [[SCALAR_PH:.*]]:
; PRED-NEXT: br label %[[LOOP:.*]]
; PRED: [[LOOP]]:
; PRED-NEXT: [[IV_1:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
@@ -750,7 +706,7 @@ define void @exit_cond_zext_iv_store64(ptr %dst, i64 %N) {
; PRED-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1
; PRED-NEXT: [[IV_EXT]] = zext i32 [[IV_1_NEXT]] to i64
; PRED-NEXT: [[C:%.*]] = icmp ult i64 [[IV_EXT]], [[N]]
-; PRED-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; PRED-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
; PRED: [[EXIT]]:
; PRED-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
index 5c12dc366b43c..51d009aede3b7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalar-steps-cost.ll
@@ -77,9 +77,9 @@ exit:
define void @scalar_steps_in_replicate_region(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'scalar_steps_in_replicate_region'
; CHECK: Cost of 0 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
-; CHECK: Cost of 0 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
-; CHECK: Cost of 0 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+; CHECK: Cost of 0.5 for VF 2: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
; CHECK: Cost of 0 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
+; CHECK: Cost of 0.5 for VF 4: {{.*}} = SCALAR-STEPS {{.*}}, ir<1>, {{.*}}
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
index e7683c6423575..517041dea1a15 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
@@ -181,136 +181,23 @@ for.cond.cleanup:
define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) {
; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
-; DEFAULT-NEXT: [[ENTRY:.*:]]
-; DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
-; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i8> poison, i8 [[B]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT1]], <16 x i8> poison, <16 x i32> zeroinitializer
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <16 x i8> poison, i8 [[C]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT3]], <16 x i8> poison, <16 x i32> zeroinitializer
-; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
-; DEFAULT: [[VECTOR_BODY]]:
-; DEFAULT-NEXT: [[TMP0:%.*]] = mul <16 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>
-; DEFAULT-NEXT: [[TMP1:%.*]] = mul <16 x i8> <i8 0, i8 0, i8 1, i8 1, i8 2, i8 2, i8 3, i8 3, i8 4, i8 4, i8 5, i8 5, i8 6, i8 6, i8 7, i8 7>, [[BROADCAST_SPLAT2]]
-; DEFAULT-NEXT: [[TMP2:%.*]] = add <16 x i8> [[TMP1]], [[TMP0]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = mul <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 3, i8 3, i8 3, i8 3>, [[BROADCAST_SPLAT4]]
-; DEFAULT-NEXT: [[TMP4:%.*]] = add <16 x i8> [[TMP2]], [[TMP3]]
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; DEFAULT: [[PRED_STORE_IF]]:
-; DEFAULT-NEXT: [[TMP5:%.*]] = extractelement <16 x i8> [[TMP4]], i64 0
-; DEFAULT-NEXT: store i8 [[TMP5]], ptr [[P]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; DEFAULT: [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; DEFAULT: [[PRED_STORE_IF5]]:
-; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
-; DEFAULT-NEXT: [[TMP7:%.*]] = extractelement <16 x i8> [[TMP4]], i64 1
-; DEFAULT-NEXT: store i8 [[TMP7]], ptr [[TMP6]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; DEFAULT: [[PRED_STORE_CONTINUE6]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; DEFAULT: [[PRED_STORE_IF7]]:
-; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; DEFAULT-NEXT: [[TMP9:%.*]] = extractelement <16 x i8> [[TMP4]], i64 2
-; DEFAULT-NEXT: store i8 [[TMP9]], ptr [[TMP8]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; DEFAULT: [[PRED_STORE_CONTINUE8]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; DEFAULT: [[PRED_STORE_IF9]]:
-; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
-; DEFAULT-NEXT: [[TMP11:%.*]] = extractelement <16 x i8> [[TMP4]], i64 3
-; DEFAULT-NEXT: store i8 [[TMP11]], ptr [[TMP10]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; DEFAULT: [[PRED_STORE_CONTINUE10]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; DEFAULT: [[PRED_STORE_IF11]]:
-; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
-; DEFAULT-NEXT: [[TMP13:%.*]] = extractelement <16 x i8> [[TMP4]], i64 4
-; DEFAULT-NEXT: store i8 [[TMP13]], ptr [[TMP12]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; DEFAULT: [[PRED_STORE_CONTINUE12]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; DEFAULT: [[PRED_STORE_IF13]]:
-; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
-; DEFAULT-NEXT: [[TMP15:%.*]] = extractelement <16 x i8> [[TMP4]], i64 5
-; DEFAULT-NEXT: store i8 [[TMP15]], ptr [[TMP14]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; DEFAULT: [[PRED_STORE_CONTINUE14]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; DEFAULT: [[PRED_STORE_IF15]]:
-; DEFAULT-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
-; DEFAULT-NEXT: [[TMP17:%.*]] = extractelement <16 x i8> [[TMP4]], i64 6
-; DEFAULT-NEXT: store i8 [[TMP17]], ptr [[TMP16]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE16]]
-; DEFAULT: [[PRED_STORE_CONTINUE16]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
-; DEFAULT: [[PRED_STORE_IF17]]:
-; DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
-; DEFAULT-NEXT: [[TMP19:%.*]] = extractelement <16 x i8> [[TMP4]], i64 7
-; DEFAULT-NEXT: store i8 [[TMP19]], ptr [[TMP18]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE18]]
-; DEFAULT: [[PRED_STORE_CONTINUE18]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
-; DEFAULT: [[PRED_STORE_IF19]]:
-; DEFAULT-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
-; DEFAULT-NEXT: [[TMP21:%.*]] = extractelement <16 x i8> [[TMP4]], i64 8
-; DEFAULT-NEXT: store i8 [[TMP21]], ptr [[TMP20]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE20]]
-; DEFAULT: [[PRED_STORE_CONTINUE20]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
-; DEFAULT: [[PRED_STORE_IF21]]:
-; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 9
-; DEFAULT-NEXT: [[TMP23:%.*]] = extractelement <16 x i8> [[TMP4]], i64 9
-; DEFAULT-NEXT: store i8 [[TMP23]], ptr [[TMP22]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE22]]
-; DEFAULT: [[PRED_STORE_CONTINUE22]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
-; DEFAULT: [[PRED_STORE_IF23]]:
-; DEFAULT-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 10
-; DEFAULT-NEXT: [[TMP25:%.*]] = extractelement <16 x i8> [[TMP4]], i64 10
-; DEFAULT-NEXT: store i8 [[TMP25]], ptr [[TMP24]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE24]]
-; DEFAULT: [[PRED_STORE_CONTINUE24]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
-; DEFAULT: [[PRED_STORE_IF25]]:
-; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 11
-; DEFAULT-NEXT: [[TMP27:%.*]] = extractelement <16 x i8> [[TMP4]], i64 11
-; DEFAULT-NEXT: store i8 [[TMP27]], ptr [[TMP26]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE26]]
-; DEFAULT: [[PRED_STORE_CONTINUE26]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
-; DEFAULT: [[PRED_STORE_IF27]]:
-; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
-; DEFAULT-NEXT: [[TMP29:%.*]] = extractelement <16 x i8> [[TMP4]], i64 12
-; DEFAULT-NEXT: store i8 [[TMP29]], ptr [[TMP28]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE28]]
-; DEFAULT: [[PRED_STORE_CONTINUE28]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
-; DEFAULT: [[PRED_STORE_IF29]]:
-; DEFAULT-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 13
-; DEFAULT-NEXT: [[TMP31:%.*]] = extractelement <16 x i8> [[TMP4]], i64 13
-; DEFAULT-NEXT: store i8 [[TMP31]], ptr [[TMP30]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE30]]
-; DEFAULT: [[PRED_STORE_CONTINUE30]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
-; DEFAULT: [[PRED_STORE_IF31]]:
-; DEFAULT-NEXT: [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 14
-; DEFAULT-NEXT: [[TMP33:%.*]] = extractelement <16 x i8> [[TMP4]], i64 14
-; DEFAULT-NEXT: store i8 [[TMP33]], ptr [[TMP32]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; DEFAULT-NEXT: [[PRED_STORE_IF31:.*]]:
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE32:.*]]
; DEFAULT: [[PRED_STORE_CONTINUE32]]:
-; DEFAULT-NEXT: br i1 false, label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
-; DEFAULT: [[PRED_STORE_IF33]]:
-; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 15
-; DEFAULT-NEXT: [[TMP35:%.*]] = extractelement <16 x i8> [[TMP4]], i64 15
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF31]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE32]] ]
+; DEFAULT-NEXT: [[TMP0:%.*]] = trunc nuw nsw i64 [[IV]] to i8
+; DEFAULT-NEXT: [[MUL:%.*]] = mul i8 [[A]], [[TMP0]]
+; DEFAULT-NEXT: [[SHR:%.*]] = lshr i8 [[TMP0]], 1
+; DEFAULT-NEXT: [[MUL5:%.*]] = mul i8 [[SHR]], [[B]]
+; DEFAULT-NEXT: [[ADD:%.*]] = add i8 [[MUL5]], [[MUL]]
+; DEFAULT-NEXT: [[SHR7:%.*]] = lshr i8 [[TMP0]], 2
+; DEFAULT-NEXT: [[MUL9:%.*]] = mul i8 [[SHR7]], [[C]]
+; DEFAULT-NEXT: [[TMP35:%.*]] = add i8 [[ADD]], [[MUL9]]
+; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
; DEFAULT-NEXT: store i8 [[TMP35]], ptr [[TMP34]], align 1
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE34]]
-; DEFAULT: [[PRED_STORE_CONTINUE34]]:
-; DEFAULT-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; DEFAULT: [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 15
+; DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[PRED_STORE_CONTINUE32]]
; DEFAULT: [[FOR_COND_CLEANUP]]:
; DEFAULT-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll
index a5ce45952aaad..bacd6aa7e2a88 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll
@@ -50,12 +50,12 @@ define void @varying_pred_store_kept_in_replicate_region(ptr noalias %dst, ptr n
; CHECK: Cost of 3.5 for VF 2: profitable to scalarize %mul = mul i32 %l, %b
; CHECK: Cost of 0 for VF 2: REPLICATE ir<%mul> = mul ir<%l>, ir<%b>
; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%mul>, ir<%gep.dst>
-; CHECK: Cost for VF 2: 9.5 (Estimated cost per lane: 4.5)
+; CHECK: Cost for VF 2: 10 (Estimated cost per lane: 5)
; CHECK: Cost of 2 for VF 4: profitable to scalarize store i32 %mul, ptr %gep.dst, align 8
; CHECK: Cost of 8.5 for VF 4: profitable to scalarize %mul = mul i32 %l, %b
; CHECK: Cost of 0 for VF 4: REPLICATE ir<%mul> = mul ir<%l>, ir<%b>
; CHECK: Cost of 0 for VF 4: REPLICATE store ir<%mul>, ir<%gep.dst>
-; CHECK: Cost for VF 4: 15.5 (Estimated cost per lane: 3.75)
+; CHECK: Cost for VF 4: 16 (Estimated cost per lane: 4)
; CHECK: LV: Selecting VF: 4.
;
entry:
More information about the llvm-commits
mailing list