[llvm-branch-commits] [clang] 721d2cf - Revert "Revert "[llvm][IR] Extend BranchWeightMetadata to track provenance of…"
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Jun 11 09:08:42 PDT 2024
Author: Paul Kirth
Date: 2024-06-11T09:08:39-07:00
New Revision: 721d2cf442610e564543f64b98566096b65ccd90
URL: https://github.com/llvm/llvm-project/commit/721d2cf442610e564543f64b98566096b65ccd90
DIFF: https://github.com/llvm/llvm-project/commit/721d2cf442610e564543f64b98566096b65ccd90.diff
LOG: Revert "Revert "[llvm][IR] Extend BranchWeightMetadata to track provenance of…"
This reverts commit 607afa0b6375e4837fef298a798f5534e783d777.
Added:
Modified:
clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
llvm/docs/BranchWeightMetadata.rst
llvm/include/llvm/IR/MDBuilder.h
llvm/include/llvm/IR/ProfDataUtils.h
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/IR/Instruction.cpp
llvm/lib/IR/Instructions.cpp
llvm/lib/IR/MDBuilder.cpp
llvm/lib/IR/Metadata.cpp
llvm/lib/IR/ProfDataUtils.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Transforms/IPO/SampleProfile.cpp
llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
llvm/lib/Transforms/Scalar/JumpThreading.cpp
llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
llvm/lib/Transforms/Utils/Local.cpp
llvm/lib/Transforms/Utils/LoopPeel.cpp
llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
llvm/lib/Transforms/Utils/SimplifyCFG.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll
llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll
llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
Removed:
################################################################################
diff --git a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
index fb236aeb982e0..81d9334356520 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp
@@ -221,5 +221,5 @@ void tu2(int &i) {
}
}
-// CHECK: [[BW_LIKELY]] = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000}
+// CHECK: [[BW_LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
+// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/llvm/docs/BranchWeightMetadata.rst b/llvm/docs/BranchWeightMetadata.rst
index 522f37cdad4fc..62204753e29b0 100644
--- a/llvm/docs/BranchWeightMetadata.rst
+++ b/llvm/docs/BranchWeightMetadata.rst
@@ -28,11 +28,14 @@ Supported Instructions
Metadata is only assigned to the conditional branches. There are two extra
operands for the true and the false branch.
+We optionally track if the metadata was added by ``__builtin_expect`` or
+``__builtin_expect_with_probability`` with an optional field ``!"expected"``.
.. code-block:: none
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <TRUE_BRANCH_WEIGHT>,
i32 <FALSE_BRANCH_WEIGHT>
}
@@ -47,6 +50,7 @@ is always case #0).
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <DEFAULT_BRANCH_WEIGHT>
[ , i32 <CASE_BRANCH_WEIGHT> ... ]
}
@@ -60,6 +64,7 @@ Branch weights are assigned to every destination.
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <LABEL_BRANCH_WEIGHT>
[ , i32 <LABEL_BRANCH_WEIGHT> ... ]
}
@@ -75,6 +80,7 @@ block and entry counts which may not be accurate with sampling.
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <CALL_BRANCH_WEIGHT>
}
@@ -95,6 +101,7 @@ is used.
!0 = !{
!"branch_weights",
+ [ !"expected", ]
i32 <INVOKE_NORMAL_WEIGHT>
[ , i32 <INVOKE_UNWIND_WEIGHT> ]
}
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index 3265589b7c8df..e02ec8f5a3d8b 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -59,7 +59,11 @@ class MDBuilder {
//===------------------------------------------------------------------===//
/// Return metadata containing two branch weights.
- MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight);
+ /// @param TrueWeight the weight of the true branch
+ /// @param FalseWeight the weight of the false branch
+ /// @param Do these weights come from __builtin_expect*
+ MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight,
+ bool IsExpected = false);
/// Return metadata containing two branch weights, with significant bias
/// towards `true` destination.
@@ -70,7 +74,10 @@ class MDBuilder {
MDNode *createUnlikelyBranchWeights();
/// Return metadata containing a number of branch weights.
- MDNode *createBranchWeights(ArrayRef<uint32_t> Weights);
+ /// @param Weights the weights of all the branches
+ /// @param Do these weights come from __builtin_expect*
+ MDNode *createBranchWeights(ArrayRef<uint32_t> Weights,
+ bool IsExpected = false);
/// Return metadata specifying that a branch or switch is unpredictable.
MDNode *createUnpredictable();
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index 88fbad4d6b9d8..1d7c97d9be953 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -55,6 +55,17 @@ MDNode *getBranchWeightMDNode(const Instruction &I);
/// Nullptr otherwise.
MDNode *getValidBranchWeightMDNode(const Instruction &I);
+/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
+/// intrinsic
+bool hasBranchWeightOrigin(const Instruction &I);
+
+/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
+/// intrinsic
+bool hasBranchWeightOrigin(const MDNode *ProfileData);
+
+/// Return the offset to the first branch weight data
+unsigned getBranchWeightOffset(const MDNode *ProfileData);
+
/// Extract branch weights from MD_prof metadata
///
/// \param ProfileData A pointer to an MDNode.
@@ -111,7 +122,11 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights);
/// Create a new `branch_weights` metadata node and add or overwrite
/// a `prof` metadata reference to instruction `I`.
-void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights);
+/// \param I the Instruction to set branch weights on.
+/// \param Weights an array of weights to set on instruction I.
+/// \param IsExpected were these weights added from an llvm.expect* intrinsic.
+void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
+ bool IsExpected);
/// Scaling the profile data attached to 'I' using the ratio of S/T.
void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 339a1f1f2f002..0e01080bd75cc 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -8866,7 +8866,8 @@ bool CodeGenPrepare::splitBranchCondition(Function &F, ModifyDT &ModifiedDT) {
scaleWeights(NewTrueWeight, NewFalseWeight);
Br1->setMetadata(LLVMContext::MD_prof,
MDBuilder(Br1->getContext())
- .createBranchWeights(TrueWeight, FalseWeight));
+ .createBranchWeights(TrueWeight, FalseWeight,
+ hasBranchWeightOrigin(*Br1)));
NewTrueWeight = TrueWeight;
NewFalseWeight = 2 * FalseWeight;
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 29272e627a1d1..aec927a8cf31d 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -1268,12 +1268,23 @@ Instruction *Instruction::cloneImpl() const {
void Instruction::swapProfMetadata() {
MDNode *ProfileData = getBranchWeightMDNode(*this);
- if (!ProfileData || ProfileData->getNumOperands() != 3)
+ if (!ProfileData)
+ return;
+ unsigned FirstIdx = getBranchWeightOffset(ProfileData);
+ if (ProfileData->getNumOperands() != 2 + FirstIdx)
return;
- // The first operand is the name. Fetch them backwards and build a new one.
- Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2),
- ProfileData->getOperand(1)};
+ unsigned SecondIdx = FirstIdx + 1;
+ SmallVector<Metadata *, 4> Ops;
+ // If there are more weights past the second, we can't swap them
+ if (ProfileData->getNumOperands() > SecondIdx + 1)
+ return;
+ for (unsigned Idx = 0; Idx < FirstIdx; ++Idx) {
+ Ops.push_back(ProfileData->getOperand(Idx));
+ }
+ // Switch the order of the weights
+ Ops.push_back(ProfileData->getOperand(SecondIdx));
+ Ops.push_back(ProfileData->getOperand(FirstIdx));
setMetadata(LLVMContext::MD_prof,
MDNode::get(ProfileData->getContext(), Ops));
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 1213f078d05ec..de369bd62a617 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -5199,7 +5199,11 @@ void SwitchInstProfUpdateWrapper::init() {
if (!ProfileData)
return;
- if (ProfileData->getNumOperands() != SI.getNumSuccessors() + 1) {
+ // FIXME: This check belongs in ProfDataUtils. Its almost equivalent to
+ // getValidBranchWeightMDNode(), but the need to use llvm_unreachable
+ // makes them slightly
diff erent.
+ if (ProfileData->getNumOperands() !=
+ SI.getNumSuccessors() + getBranchWeightOffset(ProfileData)) {
llvm_unreachable("number of prof branch_weights metadata operands does "
"not correspond to number of succesors");
}
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index bd68db3a6f961..000027754d13e 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -35,8 +35,8 @@ MDNode *MDBuilder::createFPMath(float Accuracy) {
}
MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight,
- uint32_t FalseWeight) {
- return createBranchWeights({TrueWeight, FalseWeight});
+ uint32_t FalseWeight, bool IsExpected) {
+ return createBranchWeights({TrueWeight, FalseWeight}, IsExpected);
}
MDNode *MDBuilder::createLikelyBranchWeights() {
@@ -49,15 +49,19 @@ MDNode *MDBuilder::createUnlikelyBranchWeights() {
return createBranchWeights(1, (1U << 20) - 1);
}
-MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights) {
+MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights,
+ bool IsExpected) {
assert(Weights.size() >= 1 && "Need at least one branch weights!");
- SmallVector<Metadata *, 4> Vals(Weights.size() + 1);
+ unsigned int Offset = IsExpected ? 2 : 1;
+ SmallVector<Metadata *, 4> Vals(Weights.size() + Offset);
Vals[0] = createString("branch_weights");
+ if (IsExpected)
+ Vals[1] = createString("expected");
Type *Int32Ty = Type::getInt32Ty(Context);
for (unsigned i = 0, e = Weights.size(); i != e; ++i)
- Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i]));
+ Vals[i + Offset] = createConstant(ConstantInt::get(Int32Ty, Weights[i]));
return MDNode::get(Context, Vals);
}
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index b6c932495a145..5f42ce22f72fe 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -1196,10 +1196,10 @@ MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
StringRef AProfName = AMDS->getString();
StringRef BProfName = BMDS->getString();
if (AProfName == "branch_weights" && BProfName == "branch_weights") {
- ConstantInt *AInstrWeight =
- mdconst::dyn_extract<ConstantInt>(A->getOperand(1));
- ConstantInt *BInstrWeight =
- mdconst::dyn_extract<ConstantInt>(B->getOperand(1));
+ ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
+ A->getOperand(getBranchWeightOffset(A)));
+ ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
+ B->getOperand(getBranchWeightOffset(B)));
assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
return MDNode::get(Ctx,
{MDHelper.createString("branch_weights"),
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index 51e78dc5e6c00..c4b1ed55de8a2 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -40,9 +40,6 @@ namespace {
// We maintain some constants here to ensure that we access the branch weights
// correctly, and can change the behavior in the future if the layout changes
-// The index at which the weights vector starts
-constexpr unsigned WeightsIdx = 1;
-
// the minimum number of operands for MD_prof nodes with branch weights
constexpr unsigned MinBWOps = 3;
@@ -75,6 +72,7 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
assert(isBranchWeightMD(ProfileData) && "wrong metadata");
unsigned NOps = ProfileData->getNumOperands();
+ unsigned WeightsIdx = getBranchWeightOffset(ProfileData);
assert(WeightsIdx < NOps && "Weights Index must be less than NOps.");
Weights.resize(NOps - WeightsIdx);
@@ -82,8 +80,8 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
ConstantInt *Weight =
mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));
assert(Weight && "Malformed branch_weight in MD_prof node");
- assert(Weight->getValue().getActiveBits() <= 32 &&
- "Too many bits for uint32_t");
+ assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) &&
+ "Too many bits for MD_prof branch_weight");
Weights[Idx - WeightsIdx] = Weight->getZExtValue();
}
}
@@ -123,6 +121,26 @@ bool hasValidBranchWeightMD(const Instruction &I) {
return getValidBranchWeightMDNode(I);
}
+bool hasBranchWeightOrigin(const Instruction &I) {
+ auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
+ return hasBranchWeightOrigin(ProfileData);
+}
+
+bool hasBranchWeightOrigin(const MDNode *ProfileData) {
+ if (!isBranchWeightMD(ProfileData))
+ return false;
+ auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(1));
+ // NOTE: if we ever have more types of branch weight provenance,
+ // we need to check the string value is "expected". For now, we
+ // supply a more generic API, and avoid the spurious comparisons.
+ assert(ProfDataName == nullptr || ProfDataName->getString() == "expected");
+ return ProfDataName != nullptr;
+}
+
+unsigned getBranchWeightOffset(const MDNode *ProfileData) {
+ return hasBranchWeightOrigin(ProfileData) ? 2 : 1;
+}
+
MDNode *getBranchWeightMDNode(const Instruction &I) {
auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
if (!isBranchWeightMD(ProfileData))
@@ -132,7 +150,9 @@ MDNode *getBranchWeightMDNode(const Instruction &I) {
MDNode *getValidBranchWeightMDNode(const Instruction &I) {
auto *ProfileData = getBranchWeightMDNode(I);
- if (ProfileData && ProfileData->getNumOperands() == 1 + I.getNumSuccessors())
+ auto Offset = getBranchWeightOffset(ProfileData);
+ if (ProfileData &&
+ ProfileData->getNumOperands() == Offset + I.getNumSuccessors())
return ProfileData;
return nullptr;
}
@@ -191,7 +211,8 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
return false;
if (ProfDataName->getString() == "branch_weights") {
- for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx++) {
+ unsigned Offset = getBranchWeightOffset(ProfileData);
+ for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) {
auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));
assert(V && "Malformed branch_weight in MD_prof node");
TotalVal += V->getValue().getZExtValue();
@@ -212,9 +233,10 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) {
return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);
}
-void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights) {
+void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
+ bool IsExpected) {
MDBuilder MDB(I.getContext());
- MDNode *BranchWeights = MDB.createBranchWeights(Weights);
+ MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected);
I.setMetadata(LLVMContext::MD_prof, BranchWeights);
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index e5927203f33a2..fe2253dd04eb3 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -104,6 +104,7 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/ModuleSlotTracker.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
@@ -4808,8 +4809,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
// Check consistency of !prof branch_weights metadata.
if (ProfName == "branch_weights") {
+ unsigned int Offset = getBranchWeightOffset(MD);
if (isa<InvokeInst>(&I)) {
- Check(MD->getNumOperands() == 2 || MD->getNumOperands() == 3,
+ Check(MD->getNumOperands() == (1 + Offset) ||
+ MD->getNumOperands() == (2 + Offset),
"Wrong number of InvokeInst branch_weights operands", MD);
} else {
unsigned ExpectedNumOperands = 0;
@@ -4829,10 +4832,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
CheckFailed("!prof branch_weights are not allowed for this instruction",
MD);
- Check(MD->getNumOperands() == 1 + ExpectedNumOperands,
+ Check(MD->getNumOperands() == Offset + ExpectedNumOperands,
"Wrong number of operands", MD);
}
- for (unsigned i = 1; i < MD->getNumOperands(); ++i) {
+ for (unsigned i = Offset; i < MD->getNumOperands(); ++i) {
auto &MDO = MD->getOperand(i);
Check(MDO, "second operand should not be null", MD);
Check(mdconst::dyn_extract<ConstantInt>(MDO),
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 92ad4c34da6e7..7e6a8817b7a67 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -1662,7 +1662,8 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
else if (OverwriteExistingWeights)
I.setMetadata(LLVMContext::MD_prof, nullptr);
} else if (!isa<IntrinsicInst>(&I)) {
- setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])});
+ setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])},
+ /*IsExpected=*/false);
}
}
} else if (OverwriteExistingWeights || ProfileSampleBlockAccurate) {
@@ -1673,7 +1674,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
if (cast<CallBase>(I).isIndirectCall()) {
I.setMetadata(LLVMContext::MD_prof, nullptr);
} else {
- setBranchWeights(I, {uint32_t(0)});
+ setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false);
}
}
}
@@ -1756,7 +1757,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
if (MaxWeight > 0 &&
(!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) {
LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n");
- setBranchWeights(*TI, Weights);
+ setBranchWeights(*TI, Weights, /*IsExpected=*/false);
ORE->emit([&]() {
return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst)
<< "most popular destination for conditional branches at "
diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index 0a3d8d6000cf4..731104d4fcef0 100644
--- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -1878,7 +1878,7 @@ void CHR::fixupBranchesAndSelects(CHRScope *Scope,
static_cast<uint32_t>(CHRBranchBias.scale(1000)),
static_cast<uint32_t>(CHRBranchBias.getCompl().scale(1000)),
};
- setBranchWeights(*MergedBR, Weights);
+ setBranchWeights(*MergedBR, Weights, /*IsExpected=*/false);
CHR_DEBUG(dbgs() << "CHR branch bias " << Weights[0] << ":" << Weights[1]
<< "\n");
}
diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 23a7c6a20aecb..6db76ca78b218 100644
--- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -259,7 +259,8 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee,
promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights);
if (AttachProfToDirectCall) {
- setBranchWeights(NewInst, {static_cast<uint32_t>(Count)});
+ setBranchWeights(NewInst, {static_cast<uint32_t>(Count)},
+ /*IsExpected=*/false);
}
using namespace ore;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 2269c2e0fffae..ac6d3348b3db9 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1474,7 +1474,8 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) {
for (auto *Succ : successors(&BB))
Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
if (Weights.size() >= 2)
- llvm::setBranchWeights(*BB.getTerminator(), Weights);
+ llvm::setBranchWeights(*BB.getTerminator(), Weights,
+ /*IsExpected=*/false);
}
unsigned NumCorruptCoverage = 0;
@@ -2260,7 +2261,7 @@ void llvm::setProfMetadata(Module *M, Instruction *TI,
misexpect::checkExpectAnnotations(*TI, Weights, /*IsFrontend=*/false);
- setBranchWeights(*TI, Weights);
+ setBranchWeights(*TI, Weights, /*IsExpected=*/false);
if (EmitBranchProbability) {
std::string BrCondStr = getBranchCondString(TI);
if (BrCondStr.empty())
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 74a8f1958dfe9..b9583836aea06 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -231,7 +231,7 @@ static void updatePredecessorProfileMetadata(PHINode *PN, BasicBlock *BB) {
Weights[0] = BP.getCompl().getNumerator();
Weights[1] = BP.getNumerator();
}
- setBranchWeights(*PredBr, Weights);
+ setBranchWeights(*PredBr, Weights, hasBranchWeightOrigin(*PredBr));
}
}
@@ -2618,7 +2618,7 @@ void JumpThreadingPass::updateBlockFreqAndEdgeWeight(BasicBlock *PredBB,
Weights.push_back(Prob.getNumerator());
auto TI = BB->getTerminator();
- setBranchWeights(*TI, Weights);
+ setBranchWeights(*TI, Weights, hasBranchWeightOrigin(*TI));
}
}
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index 6f87e4d91d2c7..17c5a4ee1fd0b 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -102,7 +102,7 @@ static bool handleSwitchExpect(SwitchInst &SI) {
misexpect::checkExpectAnnotations(SI, Weights, /*IsFrontend=*/true);
SI.setCondition(ArgValue);
- setBranchWeights(SI, Weights);
+ setBranchWeights(SI, Weights, /*IsExpected=*/true);
return true;
}
@@ -262,11 +262,13 @@ static void handlePhiDef(CallInst *Expect) {
if (IsOpndComingFromSuccessor(BI->getSuccessor(1)))
BI->setMetadata(LLVMContext::MD_prof,
MDB.createBranchWeights(LikelyBranchWeightVal,
- UnlikelyBranchWeightVal));
+ UnlikelyBranchWeightVal,
+ /*IsExpected=*/true));
else if (IsOpndComingFromSuccessor(BI->getSuccessor(0)))
BI->setMetadata(LLVMContext::MD_prof,
MDB.createBranchWeights(UnlikelyBranchWeightVal,
- LikelyBranchWeightVal));
+ LikelyBranchWeightVal,
+ /*IsExpected=*/true));
}
}
@@ -331,12 +333,12 @@ template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
SmallVector<uint32_t, 4> ExpectedWeights;
if ((ExpectedValue->getZExtValue() == ValueComparedTo) ==
(Predicate == CmpInst::ICMP_EQ)) {
- Node =
- MDB.createBranchWeights(LikelyBranchWeightVal, UnlikelyBranchWeightVal);
+ Node = MDB.createBranchWeights(
+ LikelyBranchWeightVal, UnlikelyBranchWeightVal, /*IsExpected=*/true);
ExpectedWeights = {LikelyBranchWeightVal, UnlikelyBranchWeightVal};
} else {
- Node =
- MDB.createBranchWeights(UnlikelyBranchWeightVal, LikelyBranchWeightVal);
+ Node = MDB.createBranchWeights(UnlikelyBranchWeightVal,
+ LikelyBranchWeightVal, /*IsExpected=*/true);
ExpectedWeights = {UnlikelyBranchWeightVal, LikelyBranchWeightVal};
}
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index ce0f4c7668a40..12229123675e7 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -231,7 +231,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
// Remove weight for this case.
std::swap(Weights[Idx + 1], Weights.back());
Weights.pop_back();
- setBranchWeights(*SI, Weights);
+ setBranchWeights(*SI, Weights, hasBranchWeightOrigin(MD));
}
// Remove this entry.
BasicBlock *ParentBB = SI->getParent();
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index e2516930d251b..d517ec3d6e9f2 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -680,7 +680,7 @@ struct WeightInfo {
/// To avoid dealing with division rounding we can just multiple both part
/// of weights to E and use weight as (F - I * E, E).
static void updateBranchWeights(Instruction *Term, WeightInfo &Info) {
- setBranchWeights(*Term, Info.Weights);
+ setBranchWeights(*Term, Info.Weights, /*IsExpected=*/false);
for (auto [Idx, SubWeight] : enumerate(Info.SubWeights))
if (SubWeight != 0)
// Don't set the probability of taking the edge from latch to loop header
@@ -1073,7 +1073,7 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI,
}
for (const auto &[Term, Info] : Weights) {
- setBranchWeights(*Term, Info.Weights);
+ setBranchWeights(*Term, Info.Weights, /*IsExpected=*/false);
}
// Update Metadata for count of peeled off iterations.
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 3d950b151cd32..04042e71a2b82 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -390,13 +390,13 @@ static void updateBranchWeights(BranchInst &PreHeaderBI, BranchInst &LoopBI,
SuccsSwapped ? LoopBackWeight : ExitWeight1,
SuccsSwapped ? ExitWeight1 : LoopBackWeight,
};
- setBranchWeights(LoopBI, LoopBIWeights);
+ setBranchWeights(LoopBI, LoopBIWeights, /*IsExpected=*/false);
if (HasConditionalPreHeader) {
const uint32_t PreHeaderBIWeights[] = {
SuccsSwapped ? EnterWeight : ExitWeight0,
SuccsSwapped ? ExitWeight0 : EnterWeight,
};
- setBranchWeights(PreHeaderBI, PreHeaderBIWeights);
+ setBranchWeights(PreHeaderBI, PreHeaderBIWeights, /*IsExpected=*/false);
}
}
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 292739b6c5fda..107c8bb6c027f 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -861,26 +861,28 @@ static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
// Set branch weights on SwitchInst. This sets the metadata if there is at
// least one non-zero weight.
-static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) {
+static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights,
+ bool IsExpected) {
// Check that there is at least one non-zero weight. Otherwise, pass
// nullptr to setMetadata which will erase the existing metadata.
MDNode *N = nullptr;
if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
- N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights);
+ N = MDBuilder(SI->getParent()->getContext())
+ .createBranchWeights(Weights, IsExpected);
SI->setMetadata(LLVMContext::MD_prof, N);
}
// Similar to the above, but for branch and select instructions that take
// exactly 2 weights.
static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
- uint32_t FalseWeight) {
+ uint32_t FalseWeight, bool IsExpected) {
assert(isa<BranchInst>(I) || isa<SelectInst>(I));
// Check that there is at least one non-zero weight. Otherwise, pass
// nullptr to setMetadata which will erase the existing metadata.
MDNode *N = nullptr;
if (TrueWeight || FalseWeight)
N = MDBuilder(I->getParent()->getContext())
- .createBranchWeights(TrueWeight, FalseWeight);
+ .createBranchWeights(TrueWeight, FalseWeight, IsExpected);
I->setMetadata(LLVMContext::MD_prof, N);
}
@@ -1338,7 +1340,7 @@ bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
- setBranchWeights(NewSI, MDWeights);
+ setBranchWeights(NewSI, MDWeights, /*IsExpected=*/false);
}
EraseTerminatorAndDCECond(PTI);
@@ -3831,7 +3833,7 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
FitWeights(NewWeights);
SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end());
- setBranchWeights(PBI, MDWeights[0], MDWeights[1]);
+ setBranchWeights(PBI, MDWeights[0], MDWeights[1], /*IsExpected=*/false);
// TODO: If BB is reachable from all paths through PredBlock, then we
// could replace PBI's branch probabilities with BI's.
@@ -4568,7 +4570,7 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
// Halve the weights if any of them cannot fit in an uint32_t
FitWeights(NewWeights);
- setBranchWeights(PBI, NewWeights[0], NewWeights[1]);
+ setBranchWeights(PBI, NewWeights[0], NewWeights[1], /*IsExpected=*/false);
}
// OtherDest may have phi nodes. If so, add an entry from PBI's
@@ -4604,7 +4606,8 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
FitWeights(NewWeights);
- setBranchWeights(NV, NewWeights[0], NewWeights[1]);
+ setBranchWeights(NV, NewWeights[0], NewWeights[1],
+ /*IsExpected=*/false);
}
}
}
@@ -4667,7 +4670,7 @@ bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
// Create a conditional branch sharing the condition of the select.
BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
if (TrueWeight != FalseWeight)
- setBranchWeights(NewBI, TrueWeight, FalseWeight);
+ setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false);
}
} else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
// Neither of the selected blocks were successors, so this
@@ -5617,7 +5620,7 @@ bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI,
TrueWeight /= 2;
FalseWeight /= 2;
}
- setBranchWeights(NewBI, TrueWeight, FalseWeight);
+ setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c7c19ef456c7c..1acecf2738b5f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2145,7 +2145,7 @@ class GeneratedRTChecks {
BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, Cond);
if (AddBranchWeights)
- setBranchWeights(BI, SCEVCheckBypassWeights);
+ setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false);
ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI);
return SCEVCheckBlock;
}
@@ -2173,7 +2173,7 @@ class GeneratedRTChecks {
BranchInst &BI =
*BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond);
if (AddBranchWeights) {
- setBranchWeights(BI, MemCheckBypassWeights);
+ setBranchWeights(BI, MemCheckBypassWeights, /*IsExpected=*/false);
}
ReplaceInstWithInst(MemCheckBlock->getTerminator(), &BI);
MemCheckBlock->getTerminator()->setDebugLoc(
@@ -2889,7 +2889,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
BranchInst &BI =
*BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters);
if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
- setBranchWeights(BI, MinItersBypassWeights);
+ setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
LoopBypassBlocks.push_back(TCCheckBlock);
}
@@ -3128,7 +3128,7 @@ BasicBlock *InnerLoopVectorizer::completeLoopSkeleton() {
unsigned TripCount = UF * VF.getKnownMinValue();
assert(TripCount > 0 && "trip count should not be zero");
const uint32_t Weights[] = {1, TripCount - 1};
- setBranchWeights(BI, Weights);
+ setBranchWeights(BI, Weights, /*IsExpected=*/false);
}
}
@@ -7669,7 +7669,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
BranchInst &BI =
*BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters);
if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
- setBranchWeights(BI, MinItersBypassWeights);
+ setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
return TCCheckBlock;
@@ -7826,7 +7826,7 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep);
const uint32_t Weights[] = {EstimatedSkipCount,
MainLoopStep - EstimatedSkipCount};
- setBranchWeights(BI, Weights);
+ setBranchWeights(BI, Weights, /*IsExpected=*/false);
}
ReplaceInstWithInst(Insert->getTerminator(), &BI);
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
index 0abca5b383224..8e06cd57a1001 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
@@ -284,7 +284,7 @@ define i32 @test10(i64 %t6) {
declare i1 @llvm.expect.i1(i1, i1) nounwind readnone
-; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1}
-; CHECK: !1 = !{!"branch_weights", i32 1, i32 2000}
-; CHECK: !2 = !{!"branch_weights", i32 1, i32 1, i32 2000}
-; CHECK: !3 = !{!"branch_weights", i32 2000, i32 1, i32 1}
+; CHECK: !0 = !{!"branch_weights", !"expected", i32 2000, i32 1}
+; CHECK: !1 = !{!"branch_weights", !"expected", i32 1, i32 2000}
+; CHECK: !2 = !{!"branch_weights", !"expected", i32 1, i32 1, i32 2000}
+; CHECK: !3 = !{!"branch_weights", !"expected", i32 2000, i32 1, i32 1}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll
index 64293557050c1..40571278ca93f 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect-with-probability.ll
@@ -284,7 +284,7 @@ define i32 @test10(i64 %t6) {
declare i1 @llvm.expect.with.probability.i1(i1, i1, double) nounwind readnone
-; CHECK: !0 = !{!"branch_weights", i32 1717986918, i32 429496731}
-; CHECK: !1 = !{!"branch_weights", i32 429496731, i32 1717986918}
-; CHECK: !2 = !{!"branch_weights", i32 214748366, i32 214748366, i32 1717986918}
-; CHECK: !3 = !{!"branch_weights", i32 1717986918, i32 214748366, i32 214748366}
+; CHECK: !0 = !{!"branch_weights", !"expected", i32 1717986918, i32 429496731}
+; CHECK: !1 = !{!"branch_weights", !"expected", i32 429496731, i32 1717986918}
+; CHECK: !2 = !{!"branch_weights", !"expected", i32 214748366, i32 214748366, i32 1717986918}
+; CHECK: !3 = !{!"branch_weights", !"expected", i32 1717986918, i32 214748366, i32 214748366}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
index 2bcfb1e064be9..458a7758fa970 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
@@ -99,6 +99,5 @@ attributes #1 = { nounwind readnone }
!0 = !{i32 1, !"wchar_size", i32 4}
!1 = !{!"clang version 5.0.0 (trunk 304373)"}
-; CHECK: [[LIKELY]] = !{!"branch_weights", i32 2000, i32 1}
-; CHECK: [[UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000}
-
+; CHECK: [[LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
+; CHECK: [[UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
index 32ae9b0b2f15c..9b9d9a746dd32 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
@@ -352,5 +352,5 @@ declare i64 @llvm.expect.i64(i64, i64)
!llvm.ident = !{!0}
!0 = !{!"clang version 5.0.0 (trunk 302965)"}
-; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1}
-; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000}
+; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
+; CHECK: [[WEIGHT2]] = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
index 1efa63241c2c0..e9a843225993a 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
@@ -99,5 +99,5 @@ declare i64 @llvm.expect.i64(i64, i64)
!0 = !{!"clang version 5.0.0 (trunk 302965)"}
-; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1}
-; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000}
+; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
+; CHECK: [[WEIGHT2]] = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
index 9cbaca8d13dc0..13db2c394bab2 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
@@ -53,4 +53,4 @@ declare i64 @llvm.expect.i64(i64, i64)
!0 = !{!"clang version 5.0.0 (trunk 302965)"}
-; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 1, i32 2000}
+; CHECK: [[WEIGHT]] = !{!"branch_weights", !"expected", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll
index 2bad66343b761..275731d618895 100644
--- a/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_unexpect.ll
@@ -235,5 +235,5 @@ block5:
ret void
}
-; CHECK: !0 = !{!"branch_weights", i32 2147483647, i32 1}
-; CHECK: !1 = !{!"branch_weights", i32 1, i32 2147483647}
+; CHECK: !0 = !{!"branch_weights", !"expected", i32 2147483647, i32 1}
+; CHECK: !1 = !{!"branch_weights", !"expected", i32 1, i32 2147483647}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
index 55dd28b70170b..a090fe20e1d93 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
@@ -293,7 +293,7 @@ declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[PROF4]] = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: [[PROF4]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
;.
More information about the llvm-branch-commits
mailing list