[llvm-branch-commits] [llvm][IR] Extend BranchWeightMetadata to track provenance of weights (PR #86609)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Mar 25 17:49:55 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-pgo
Author: Paul Kirth (ilovepi)
<details>
<summary>Changes</summary>
This patch implements the changes to LLVM IR discussed in
https://discourse.llvm.org/t/rfc-update-branch-weights-metadata-to-allow-tracking-branch-weight-origins/75032
In this patch, we add an optional field to MD_prof meatdata nodes for
branch weights, which can be used to distinguish weights added from
`llvm.expect*` intrinsics from those added via other methods, e.g.
from profiles or inserted by the compiler.
One of the major motivations, is for use with MisExpect diagnostics,
which need to know if branch_weight metadata originates from an
llvm.expect intrinsic. Without that information, we end up checking
branch weights multiple times in the case if ThinLTO + SampleProfiling,
leading to some inaccuracy in how we report MisExpect related
diagnostics to users.
Since we change the format of MD_prof metadata in a fundamental way, we
need to update code handling branch weights in a number of places.
We also update the lang ref for branch weights to reflect the change.
---
Patch is 37.48 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/86609.diff
28 Files Affected:
- (modified) clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp (+2-2)
- (modified) llvm/docs/BranchWeightMetadata.rst (+7)
- (modified) llvm/include/llvm/IR/MDBuilder.h (+9-2)
- (modified) llvm/include/llvm/IR/ProfDataUtils.h (+18-1)
- (modified) llvm/lib/CodeGen/CodeGenPrepare.cpp (+2-1)
- (modified) llvm/lib/IR/Instruction.cpp (+14-4)
- (modified) llvm/lib/IR/Instructions.cpp (+10-4)
- (modified) llvm/lib/IR/MDBuilder.cpp (+9-5)
- (modified) llvm/lib/IR/ProfDataUtils.cpp (+34-14)
- (modified) llvm/lib/IR/Verifier.cpp (+6-3)
- (modified) llvm/lib/Transforms/IPO/SampleProfile.cpp (+4-3)
- (modified) llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp (+1-1)
- (modified) llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp (+2-1)
- (modified) llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp (+2-2)
- (modified) llvm/lib/Transforms/Scalar/JumpThreading.cpp (+2-2)
- (modified) llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp (+9-7)
- (modified) llvm/lib/Transforms/Utils/Local.cpp (+1-1)
- (modified) llvm/lib/Transforms/Utils/LoopPeel.cpp (+2-2)
- (modified) llvm/lib/Transforms/Utils/LoopRotationUtils.cpp (+2-2)
- (modified) llvm/lib/Transforms/Utils/SimplifyCFG.cpp (+10-10)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+6-6)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/basic.ll (+4-4)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll (+4-4)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll (+2-3)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll (+2-2)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll (+2-2)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll (+1-1)
- (modified) llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll (+2-2)
``````````diff
diff --git a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
index fb236aeb982e01..81d93343565209 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
@@ -221,5 +221,5 @@ void tu2(int &i) {
}
}
-// CHECK: [[BW_LIKELY]] = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000}
+// CHECK: [[BW_LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
+// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/llvm/docs/BranchWeightMetadata.rst b/llvm/docs/BranchWeightMetadata.rst
index 522f37cdad4fc1..62204753e29b06 100644
--- a/llvm/docs/BranchWeightMetadata.rst
+++ b/llvm/docs/BranchWeightMetadata.rst
@@ -28,11 +28,14 @@ Supported Instructions
Metadata is only assigned to the conditional branches. There are two extra
operands for the true and the false branch.
+We optionally track if the metadata was added by ``__builtin_expect`` or
+``__builtin_expect_with_probability`` with an optional field ``!"expected"``.
.. code-block:: none
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <TRUE_BRANCH_WEIGHT>,
i32 <FALSE_BRANCH_WEIGHT>
}
@@ -47,6 +50,7 @@ is always case #0).
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <DEFAULT_BRANCH_WEIGHT>
[ , i32 <CASE_BRANCH_WEIGHT> ... ]
}
@@ -60,6 +64,7 @@ Branch weights are assigned to every destination.
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <LABEL_BRANCH_WEIGHT>
[ , i32 <LABEL_BRANCH_WEIGHT> ... ]
}
@@ -75,6 +80,7 @@ block and entry counts which may not be accurate with sampling.
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <CALL_BRANCH_WEIGHT>
}
@@ -95,6 +101,7 @@ is used.
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <INVOKE_NORMAL_WEIGHT>
[ , i32 <INVOKE_UNWIND_WEIGHT> ]
}
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index 39165453de16b0..bc2bf50033deec 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -59,10 +59,17 @@ class MDBuilder {
//===------------------------------------------------------------------===//
/// Return metadata containing two branch weights.
- MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight);
+ /// @param TrueWeight the weight of the true branch
+ /// @param FalseWeight the weight of the false branch
+ /// @param Do these weights come from __builtin_expect*
+ MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight,
+ bool IsExpected = false);
/// Return metadata containing a number of branch weights.
- MDNode *createBranchWeights(ArrayRef<uint32_t> Weights);
+ /// @param Weights the weights of all the branches
+ /// @param Do these weights come from __builtin_expect*
+ MDNode *createBranchWeights(ArrayRef<uint32_t> Weights,
+ bool IsExpected = false);
/// Return metadata specifying that a branch or switch is unpredictable.
MDNode *createUnpredictable();
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index dc983eed13a8d3..5363042e7c110c 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -55,6 +55,20 @@ MDNode *getBranchWeightMDNode(const Instruction &I);
/// Nullptr otherwise.
MDNode *getValidBranchWeightMDNode(const Instruction &I);
+/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
+/// intrinsic
+bool hasExpectedProvenance(const Instruction &I);
+
+/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
+/// intrinsic
+bool hasExpectedProvenance(const MDNode *ProfileData);
+
+/// Return the offset to the first branch weight data
+unsigned getBranchWeightOffset(const Instruction &I);
+
+/// Return the offset to the first branch weight data
+unsigned getBranchWeightOffset(const MDNode *ProfileData);
+
/// Extract branch weights from MD_prof metadata
///
/// \param ProfileData A pointer to an MDNode.
@@ -111,7 +125,10 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights);
/// Create a new `branch_weights` metadata node and add or overwrite
/// a `prof` metadata reference to instruction `I`.
-void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights);
+/// \param I the Instruction to set branch weights on.
+/// \param Weights an array of weights to set on instruction I.
+/// \param IsExpected were these weights added from an llvm.expect* intrinsic.
+void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights, bool IsExpected);
} // namespace llvm
#endif
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 9f99bb7e693f7e..3c9cae0e8063bf 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -8843,7 +8843,8 @@ bool CodeGenPrepare::splitBranchCondition(Function &F, ModifyDT &ModifiedDT) {
scaleWeights(NewTrueWeight, NewFalseWeight);
Br1->setMetadata(LLVMContext::MD_prof,
MDBuilder(Br1->getContext())
- .createBranchWeights(TrueWeight, FalseWeight));
+ .createBranchWeights(TrueWeight, FalseWeight,
+ hasExpectedProvenance(*Br1)));
NewTrueWeight = TrueWeight;
NewFalseWeight = 2 * FalseWeight;
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 47a7f2c9de790f..0db3d917d388c8 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -1210,12 +1210,22 @@ Instruction *Instruction::cloneImpl() const {
void Instruction::swapProfMetadata() {
MDNode *ProfileData = getBranchWeightMDNode(*this);
- if (!ProfileData || ProfileData->getNumOperands() != 3)
+ if(!isBranchWeightMD(ProfileData))
return;
- // The first operand is the name. Fetch them backwards and build a new one.
- Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2),
- ProfileData->getOperand(1)};
+ SmallVector<Metadata *, 4> Ops;
+ unsigned int FirstIdx = getBranchWeightOffset(ProfileData);
+ unsigned int SecondIdx = FirstIdx + 1;
+ // If there are more weights past the second, we can't swap them
+ if(ProfileData->getNumOperands() > SecondIdx +1)
+ return;
+ Ops.push_back(ProfileData->getOperand(0));
+ if (hasExpectedProvenance(ProfileData)) {
+ Ops.push_back(ProfileData->getOperand(1));
+ }
+ // Switch the order of the weights
+ Ops.push_back(ProfileData->getOperand(SecondIdx));
+ Ops.push_back(ProfileData->getOperand(FirstIdx));
setMetadata(LLVMContext::MD_prof,
MDNode::get(ProfileData->getContext(), Ops));
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 8ca211e6e2e72f..d7c8b4d976e963 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -857,10 +857,12 @@ void CallInst::updateProfWeight(uint64_t S, uint64_t T) {
APInt APS(128, S), APT(128, T);
if (ProfDataName->getString().equals("branch_weights") &&
ProfileData->getNumOperands() > 0) {
+ unsigned int Offset = getBranchWeightOffset(ProfileData);
// Using APInt::div may be expensive, but most cases should fit 64 bits.
- APInt Val(128, mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1))
- ->getValue()
- .getZExtValue());
+ APInt Val(128,
+ mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Offset))
+ ->getValue()
+ .getZExtValue());
Val *= APS;
Vals.push_back(MDB.createConstant(
ConstantInt::get(Type::getInt32Ty(getContext()),
@@ -5196,7 +5198,11 @@ void SwitchInstProfUpdateWrapper::init() {
if (!ProfileData)
return;
- if (ProfileData->getNumOperands() != SI.getNumSuccessors() + 1) {
+ // FIXME: This check belongs in ProfDataUtils. Its almost equivalent to
+ // getValidBranchWeightMDNode(), but the need to use llvm_unreachable
+ // makes them slightly different.
+ if (ProfileData->getNumOperands() !=
+ SI.getNumSuccessors() + getBranchWeightOffset(ProfileData)) {
llvm_unreachable("number of prof branch_weights metadata operands does "
"not correspond to number of succesors");
}
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index 2490b3012bdc2b..2667e929a2e21d 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -35,19 +35,23 @@ MDNode *MDBuilder::createFPMath(float Accuracy) {
}
MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight,
- uint32_t FalseWeight) {
- return createBranchWeights({TrueWeight, FalseWeight});
+ uint32_t FalseWeight, bool IsExpected) {
+ return createBranchWeights({TrueWeight, FalseWeight}, IsExpected);
}
-MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights) {
+MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights,
+ bool IsExpected) {
assert(Weights.size() >= 1 && "Need at least one branch weights!");
- SmallVector<Metadata *, 4> Vals(Weights.size() + 1);
+ unsigned int Offset = IsExpected ? 2 : 1;
+ SmallVector<Metadata *, 4> Vals(Weights.size() + Offset);
Vals[0] = createString("branch_weights");
+ if(IsExpected)
+ Vals[1] = createString("expected");
Type *Int32Ty = Type::getInt32Ty(Context);
for (unsigned i = 0, e = Weights.size(); i != e; ++i)
- Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i]));
+ Vals[i + Offset] = createConstant(ConstantInt::get(Int32Ty, Weights[i]));
return MDNode::get(Context, Vals);
}
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index b4e09e76993f99..b48d3cdb30c48a 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -40,9 +40,6 @@ namespace {
// We maintain some constants here to ensure that we access the branch weights
// correctly, and can change the behavior in the future if the layout changes
-// The index at which the weights vector starts
-constexpr unsigned WeightsIdx = 1;
-
// the minimum number of operands for MD_prof nodes with branch weights
constexpr unsigned MinBWOps = 3;
@@ -72,6 +69,7 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
assert(isBranchWeightMD(ProfileData) && "wrong metadata");
unsigned NOps = ProfileData->getNumOperands();
+ unsigned int WeightsIdx = getBranchWeightOffset(ProfileData);
assert(WeightsIdx < NOps && "Weights Index must be less than NOps.");
Weights.resize(NOps - WeightsIdx);
@@ -79,8 +77,8 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
ConstantInt *Weight =
mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));
assert(Weight && "Malformed branch_weight in MD_prof node");
- assert(Weight->getValue().getActiveBits() <= 32 &&
- "Too many bits for uint32_t");
+ assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) &&
+ "Too many bits for MD_prof branch_weight");
Weights[Idx - WeightsIdx] = Weight->getZExtValue();
}
}
@@ -106,6 +104,30 @@ bool hasValidBranchWeightMD(const Instruction &I) {
return getValidBranchWeightMDNode(I);
}
+bool hasExpectedProvenance(const Instruction &I) {
+ auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
+ return hasExpectedProvenance(ProfileData);
+}
+
+bool hasExpectedProvenance(const MDNode *ProfileData) {
+ if (!isBranchWeightMD(ProfileData))
+ return false;
+
+ auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(1));
+ if (!ProfDataName)
+ return false;
+ return ProfDataName->getString().equals("expected");
+}
+
+unsigned getBranchWeightOffset(const Instruction &I) {
+ auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
+ return getBranchWeightOffset(ProfileData);
+}
+
+unsigned getBranchWeightOffset(const MDNode *ProfileData) {
+ return hasExpectedProvenance(ProfileData) ? 2 : 1;
+}
+
MDNode *getBranchWeightMDNode(const Instruction &I) {
auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
if (!isBranchWeightMD(ProfileData))
@@ -115,7 +137,8 @@ MDNode *getBranchWeightMDNode(const Instruction &I) {
MDNode *getValidBranchWeightMDNode(const Instruction &I) {
auto *ProfileData = getBranchWeightMDNode(I);
- if (ProfileData && ProfileData->getNumOperands() == 1 + I.getNumSuccessors())
+ auto Offset = getBranchWeightOffset(ProfileData);
+ if (ProfileData && ProfileData->getNumOperands() == Offset + I.getNumSuccessors())
return ProfileData;
return nullptr;
}
@@ -130,11 +153,6 @@ void extractFromBranchWeightMD64(const MDNode *ProfileData,
extractFromBranchWeightMD(ProfileData, Weights);
}
-
-
-
-
-
bool extractBranchWeights(const MDNode *ProfileData,
SmallVectorImpl<uint32_t> &Weights) {
if (!isBranchWeightMD(ProfileData))
@@ -179,7 +197,8 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
return false;
if (ProfDataName->getString().equals("branch_weights")) {
- for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx++) {
+ unsigned int Offset = getBranchWeightOffset(ProfileData);
+ for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) {
auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));
assert(V && "Malformed branch_weight in MD_prof node");
TotalVal += V->getValue().getZExtValue();
@@ -201,9 +220,10 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) {
return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);
}
-void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights) {
+void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
+ bool IsExpected) {
MDBuilder MDB(I.getContext());
- MDNode *BranchWeights = MDB.createBranchWeights(Weights);
+ MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected);
I.setMetadata(LLVMContext::MD_prof, BranchWeights);
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 819722566831c6..2305343306aa9c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -103,6 +103,7 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/ModuleSlotTracker.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
@@ -4756,8 +4757,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
// Check consistency of !prof branch_weights metadata.
if (ProfName.equals("branch_weights")) {
+ unsigned int Offset = getBranchWeightOffset(I);
if (isa<InvokeInst>(&I)) {
- Check(MD->getNumOperands() == 2 || MD->getNumOperands() == 3,
+ Check(MD->getNumOperands() == (1 + Offset) ||
+ MD->getNumOperands() == (2 + Offset),
"Wrong number of InvokeInst branch_weights operands", MD);
} else {
unsigned ExpectedNumOperands = 0;
@@ -4777,10 +4780,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
CheckFailed("!prof branch_weights are not allowed for this instruction",
MD);
- Check(MD->getNumOperands() == 1 + ExpectedNumOperands,
+ Check(MD->getNumOperands() == Offset + ExpectedNumOperands,
"Wrong number of operands", MD);
}
- for (unsigned i = 1; i < MD->getNumOperands(); ++i) {
+ for (unsigned i = Offset; i < MD->getNumOperands(); ++i) {
auto &MDO = MD->getOperand(i);
Check(MDO, "second operand should not be null", MD);
Check(mdconst::dyn_extract<ConstantInt>(MDO),
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 9a8040bc4b064e..5d4cacb679148b 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -1775,7 +1775,8 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
else if (OverwriteExistingWeights)
I.setMetadata(LLVMContext::MD_prof, nullptr);
} else if (!isa<IntrinsicInst>(&I)) {
- setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])});
+ setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])},
+ /*IsExpected=*/false);
}
}
} else if (OverwriteExistingWeights || ProfileSampleBlockAccurate) {
@@ -1786,7 +1787,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
if (cast<CallBase>(I).isIndirectCall()) {
I.setMetadata(LLVMContext::MD_prof, nullptr);
} else {
- setBranchWeights(I, {uint32_t(0)});
+ setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false);
}
}
}
@@ -1867,7 +1868,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
if (MaxWeight > 0 &&
(!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) {
LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n");
- setBranchWeights(*TI, Weights);
+ setBranchWeights(*TI, Weights, /*IsExpected=*/false);
ORE->emit([&]() {
return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst)
<< "most popular destination for conditional branches at "
diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index 0a3d8d6000cf47..731104d4fcef02 100644
--- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -1878,7 +1878,7 @@ void CHR::fixupBranchesAndSelects(CHRScope *Scope,
static_cast<uint32_t>(CHRBranchBias.scale(1000)),
static_cast<uint32_t>(CHRBranchBias.getCompl().scale(1000)),
};
- setBranchWeights(*MergedBR, Weights);
+ setBranchWeights(*MergedBR, Weights, /*IsExpected=*/false);
CHR_DEBUG(dbgs() << "CHR branch bias " << Weights[0] << ":" << Weights[1]
<< "\n");
}
diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 7344fea1751719..a43395702f1dcd 100644
--- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -257,7 +257,8 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee,
promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights);
if (AttachProfToDirectCall) {
- setBranchWeights(NewInst, {static_cast<uint32_t>(Count)});
+ setBranchWeights(NewInst, {static_cast<uint32_t>(Count)},
+ /*IsExpected=*/false);
}
using namespace ore;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 50eccc69a38a00..587eb82e1c6769 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1416,7 +1416,7 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) {
for (auto *Succ : successors(&BB))
Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
if (Weights.size() >= 2)
- llvm::setBranchWeights(*BB.getTerminator(), Weights);
+ llvm::setBranchWeights(*BB.getTerminator(), Weights, /*IsExpected=*/false);
}
unsigned NumCorruptCoverage = 0;
@@ -2191,7 +2191,7 @@ void llvm::setProfMetadata(Module *M, Instruction *TI,
misexpect::checkExpectAnnotations(*TI, Weights, /*IsFrontend=*/false);
- setBranchWeights(*TI, Weights);
+ setBranchWeights(*TI, Weights, /*IsExpected=*/false);
if (EmitBranchProbability) {
std::string BrCondStr = getBranchCondString(TI);
if (BrCondStr.empty())
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index ffcb...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/86609
More information about the llvm-branch-commits
mailing list