[llvm] r255455 - Normalize MBB's successors' probabilities in several locations.
Cong Hou via llvm-commits
llvm-commits at lists.llvm.org
Sun Dec 13 01:26:18 PST 2015
Author: conghou
Date: Sun Dec 13 03:26:17 2015
New Revision: 255455
URL: http://llvm.org/viewvc/llvm-project?rev=255455&view=rev
Log:
Normalize MBB's successors' probabilities in several locations.
This patch adds some missing calls to MBB::normalizeSuccProbs() in several
locations where it should be called. Those places are found by checking if the
sum of successors' probabilities is approximate one in MachineBlockPlacement
pass with some instrumented code (not in this patch).
Differential revision: http://reviews.llvm.org/D15259
Modified:
llvm/trunk/include/llvm/CodeGen/MachineBasicBlock.h
llvm/trunk/lib/CodeGen/EarlyIfConversion.cpp
llvm/trunk/lib/CodeGen/IfConversion.cpp
llvm/trunk/lib/CodeGen/MachineBasicBlock.cpp
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/trunk/lib/CodeGen/TailDuplication.cpp
llvm/trunk/lib/Target/AArch64/AArch64ConditionalCompares.cpp
llvm/trunk/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp
llvm/trunk/lib/Target/Hexagon/HexagonEarlyIfConv.cpp
llvm/trunk/lib/Target/Mips/MipsLongBranch.cpp
llvm/trunk/lib/Target/PowerPC/PPCEarlyReturn.cpp
Modified: llvm/trunk/include/llvm/CodeGen/MachineBasicBlock.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/MachineBasicBlock.h?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/MachineBasicBlock.h (original)
+++ llvm/trunk/include/llvm/CodeGen/MachineBasicBlock.h Sun Dec 13 03:26:17 2015
@@ -454,19 +454,32 @@ public:
void setSuccProbability(succ_iterator I, BranchProbability Prob);
/// Normalize probabilities of all successors so that the sum of them becomes
- /// one.
+ /// one. This is usually done when the current update on this MBB is done, and
+ /// the sum of its successors' probabilities is not guaranteed to be one. The
+ /// user is responsible for the correct use of this function.
+ /// MBB::removeSuccessor() has an option to do this automatically.
void normalizeSuccProbs() {
BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
}
+ /// Validate successors' probabilities and check if the sum of them is
+ /// approximate one. This only works in DEBUG mode.
+ void validateSuccProbs() const;
+
/// Remove successor from the successors list of this MachineBasicBlock. The
/// Predecessors list of Succ is automatically updated.
- void removeSuccessor(MachineBasicBlock *Succ);
+ /// If NormalizeSuccProbs is true, then normalize successors' probabilities
+ /// after the successor is removed.
+ void removeSuccessor(MachineBasicBlock *Succ,
+ bool NormalizeSuccProbs = false);
/// Remove specified successor from the successors list of this
/// MachineBasicBlock. The Predecessors list of Succ is automatically updated.
+ /// If NormalizeSuccProbs is true, then normalize successors' probabilities
+ /// after the successor is removed.
/// Return the iterator to the element after the one removed.
- succ_iterator removeSuccessor(succ_iterator I);
+ succ_iterator removeSuccessor(succ_iterator I,
+ bool NormalizeSuccProbs = false);
/// Replace successor OLD with NEW and update probability info.
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New);
Modified: llvm/trunk/lib/CodeGen/EarlyIfConversion.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/EarlyIfConversion.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/EarlyIfConversion.cpp (original)
+++ llvm/trunk/lib/CodeGen/EarlyIfConversion.cpp Sun Dec 13 03:26:17 2015
@@ -538,11 +538,11 @@ void SSAIfConv::convertIf(SmallVectorImp
// Fix up the CFG, temporarily leave Head without any successors.
Head->removeSuccessor(TBB);
- Head->removeSuccessor(FBB);
+ Head->removeSuccessor(FBB, true);
if (TBB != Tail)
- TBB->removeSuccessor(Tail);
+ TBB->removeSuccessor(Tail, true);
if (FBB != Tail)
- FBB->removeSuccessor(Tail);
+ FBB->removeSuccessor(Tail, true);
// Fix up Head's terminators.
// It should become a single branch or a fallthrough.
Modified: llvm/trunk/lib/CodeGen/IfConversion.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/IfConversion.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/IfConversion.cpp (original)
+++ llvm/trunk/lib/CodeGen/IfConversion.cpp Sun Dec 13 03:26:17 2015
@@ -1113,7 +1113,7 @@ bool IfConverter::IfConvertSimple(BBInfo
// RemoveExtraEdges won't work if the block has an unanalyzable branch, so
// explicitly remove CvtBBI as a successor.
- BBI.BB->removeSuccessor(CvtBBI->BB);
+ BBI.BB->removeSuccessor(CvtBBI->BB, true);
} else {
RemoveKills(CvtBBI->BB->begin(), CvtBBI->BB->end(), DontKill, *TRI);
PredicateBlock(*CvtBBI, CvtBBI->BB->end(), Cond);
@@ -1226,7 +1226,7 @@ bool IfConverter::IfConvertTriangle(BBIn
// RemoveExtraEdges won't work if the block has an unanalyzable branch, so
// explicitly remove CvtBBI as a successor.
- BBI.BB->removeSuccessor(CvtBBI->BB);
+ BBI.BB->removeSuccessor(CvtBBI->BB, true);
} else {
// Predicate the 'true' block after removing its branch.
CvtBBI->NonPredSize -= TII->RemoveBranch(*CvtBBI->BB);
@@ -1512,7 +1512,7 @@ bool IfConverter::IfConvertDiamond(BBInf
// which can happen here if TailBB is unanalyzable and is merged, so
// explicitly remove BBI1 and BBI2 as successors.
BBI.BB->removeSuccessor(BBI1->BB);
- BBI.BB->removeSuccessor(BBI2->BB);
+ BBI.BB->removeSuccessor(BBI2->BB, true);
RemoveExtraEdges(BBI);
// Update block info.
@@ -1706,7 +1706,7 @@ void IfConverter::MergeBlocks(BBInfo &To
if (AddEdges) {
// If the edge from ToBBI.BB to Succ already exists, update the
- // probability of this edge by adding NewWeight to it. An example is shown
+ // probability of this edge by adding NewProb to it. An example is shown
// below, in which A is ToBBI.BB and B is FromBBI.BB. In this case we
// don't have to set C as A's successor as it already is. We only need to
// update the edge probability on A->C. Note that B will not be
@@ -1740,6 +1740,10 @@ void IfConverter::MergeBlocks(BBInfo &To
if (NBB && !FromBBI.BB->isSuccessor(NBB))
FromBBI.BB->addSuccessor(NBB);
+ // Normalize the probabilities of ToBBI.BB's successors with all adjustment
+ // we've done above.
+ ToBBI.BB->normalizeSuccProbs();
+
ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end());
FromBBI.Predicate.clear();
Modified: llvm/trunk/lib/CodeGen/MachineBasicBlock.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/MachineBasicBlock.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/MachineBasicBlock.cpp (original)
+++ llvm/trunk/lib/CodeGen/MachineBasicBlock.cpp Sun Dec 13 03:26:17 2015
@@ -506,6 +506,20 @@ void MachineBasicBlock::updateTerminator
}
}
+void MachineBasicBlock::validateSuccProbs() const {
+#ifndef NDEBUG
+ int64_t Sum = 0;
+ for (auto Prob : Probs)
+ Sum += Prob.getNumerator();
+ // Due to precision issue, we assume that the sum of probabilities is one if
+ // the difference between the sum of their numerators and the denominator is
+ // no greater than the number of successors.
+ assert(std::abs<uint64_t>(Sum - BranchProbability::getDenominator()) <=
+ Probs.size() &&
+ "The sum of successors's probabilities exceeds one.");
+#endif // NDEBUG
+}
+
void MachineBasicBlock::addSuccessor(MachineBasicBlock *Succ,
BranchProbability Prob) {
// Probability list is either empty (if successor list isn't empty, this means
@@ -525,13 +539,14 @@ void MachineBasicBlock::addSuccessorWith
Succ->addPredecessor(this);
}
-void MachineBasicBlock::removeSuccessor(MachineBasicBlock *Succ) {
+void MachineBasicBlock::removeSuccessor(MachineBasicBlock *Succ,
+ bool NormalizeSuccProbs) {
succ_iterator I = std::find(Successors.begin(), Successors.end(), Succ);
- removeSuccessor(I);
+ removeSuccessor(I, NormalizeSuccProbs);
}
MachineBasicBlock::succ_iterator
-MachineBasicBlock::removeSuccessor(succ_iterator I) {
+MachineBasicBlock::removeSuccessor(succ_iterator I, bool NormalizeSuccProbs) {
assert(I != Successors.end() && "Not a current successor!");
// If probability list is empty it means we don't use it (disabled
@@ -539,6 +554,8 @@ MachineBasicBlock::removeSuccessor(succ_
if (!Probs.empty()) {
probability_iterator WI = getProbabilityIterator(I);
Probs.erase(WI);
+ if (NormalizeSuccProbs)
+ normalizeSuccProbs();
}
(*I)->removePredecessor(this);
@@ -636,6 +653,7 @@ MachineBasicBlock::transferSuccessorsAnd
MO.setMBB(this);
}
}
+ normalizeSuccProbs();
}
bool MachineBasicBlock::isPredecessor(const MachineBasicBlock *MBB) const {
@@ -1088,6 +1106,8 @@ bool MachineBasicBlock::CorrectExtraCFGE
}
}
+ if (Changed)
+ normalizeSuccProbs();
return Changed;
}
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Sun Dec 13 03:26:17 2015
@@ -2270,6 +2270,7 @@ void SelectionDAGBuilder::visitIndirectB
MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
addSuccessorWithProb(IndirectBrMBB, Succ);
}
+ IndirectBrMBB->normalizeSuccProbs();
DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
MVT::Other, getControlRoot(),
Modified: llvm/trunk/lib/CodeGen/TailDuplication.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/TailDuplication.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/TailDuplication.cpp (original)
+++ llvm/trunk/lib/CodeGen/TailDuplication.cpp Sun Dec 13 03:26:17 2015
@@ -751,6 +751,7 @@ TailDuplicatePass::duplicateSimpleBB(Mac
assert(NumSuccessors <= 1);
if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget)
PredBB->addSuccessor(NewTarget, Prob);
+ PredBB->normalizeSuccProbs();
TDBBs.push_back(PredBB);
}
Modified: llvm/trunk/lib/Target/AArch64/AArch64ConditionalCompares.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64ConditionalCompares.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64ConditionalCompares.cpp (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64ConditionalCompares.cpp Sun Dec 13 03:26:17 2015
@@ -567,8 +567,8 @@ void SSACCmpConv::convert(SmallVectorImp
// All CmpBB instructions are moved into Head, and CmpBB is deleted.
// Update the CFG first.
updateTailPHIs();
- Head->removeSuccessor(CmpBB);
- CmpBB->removeSuccessor(Tail);
+ Head->removeSuccessor(CmpBB, true);
+ CmpBB->removeSuccessor(Tail, true);
Head->transferSuccessorsAndUpdatePHIs(CmpBB);
DebugLoc TermDL = Head->getFirstTerminator()->getDebugLoc();
TII->RemoveBranch(*Head);
Modified: llvm/trunk/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp Sun Dec 13 03:26:17 2015
@@ -1164,7 +1164,7 @@ int AMDGPUCFGStructurizer::loopcontPatte
for (SmallVectorImpl<MachineBasicBlock *>::iterator It = ContMBB.begin(),
E = ContMBB.end(); It != E; ++It) {
- (*It)->removeSuccessor(LoopHeader);
+ (*It)->removeSuccessor(LoopHeader, true);
}
numLoopcontPatternMatch += NumCont;
@@ -1487,7 +1487,7 @@ void AMDGPUCFGStructurizer::mergeSerialB
);
DstMBB->splice(DstMBB->end(), SrcMBB, SrcMBB->begin(), SrcMBB->end());
- DstMBB->removeSuccessor(SrcMBB);
+ DstMBB->removeSuccessor(SrcMBB, true);
cloneSuccessorList(DstMBB, SrcMBB);
removeSuccessor(SrcMBB);
@@ -1537,9 +1537,9 @@ void AMDGPUCFGStructurizer::mergeIfthene
if (TrueMBB) {
MBB->splice(I, TrueMBB, TrueMBB->begin(), TrueMBB->end());
- MBB->removeSuccessor(TrueMBB);
+ MBB->removeSuccessor(TrueMBB, true);
if (LandMBB && TrueMBB->succ_size()!=0)
- TrueMBB->removeSuccessor(LandMBB);
+ TrueMBB->removeSuccessor(LandMBB, true);
retireBlock(TrueMBB);
MLI->removeBlock(TrueMBB);
}
@@ -1548,9 +1548,9 @@ void AMDGPUCFGStructurizer::mergeIfthene
insertInstrBefore(I, AMDGPU::ELSE);
MBB->splice(I, FalseMBB, FalseMBB->begin(),
FalseMBB->end());
- MBB->removeSuccessor(FalseMBB);
+ MBB->removeSuccessor(FalseMBB, true);
if (LandMBB && FalseMBB->succ_size() != 0)
- FalseMBB->removeSuccessor(LandMBB);
+ FalseMBB->removeSuccessor(LandMBB, true);
retireBlock(FalseMBB);
MLI->removeBlock(FalseMBB);
}
@@ -1591,7 +1591,7 @@ void AMDGPUCFGStructurizer::mergeLoopbre
//now branchInst can be erase safely
BranchMI->eraseFromParent();
//now take care of successors, retire blocks
- ExitingMBB->removeSuccessor(LandMBB);
+ ExitingMBB->removeSuccessor(LandMBB, true);
}
void AMDGPUCFGStructurizer::settleLoopcontBlock(MachineBasicBlock *ContingMBB,
@@ -1757,7 +1757,7 @@ void AMDGPUCFGStructurizer::removeRedund
DEBUG(dbgs() << "Removing unneeded cond branch instr: " << *BranchMI);
BranchMI->eraseFromParent();
SHOWNEWBLK(MBB1, "Removing redundant successor");
- MBB->removeSuccessor(MBB1);
+ MBB->removeSuccessor(MBB1, true);
}
void AMDGPUCFGStructurizer::addDummyExitBlock(
Modified: llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp Sun Dec 13 03:26:17 2015
@@ -7397,6 +7397,7 @@ void ARMTargetLowering::EmitSjLjDispatch
}
BB->addSuccessor(DispatchBB, BranchProbability::getZero());
+ BB->normalizeSuccProbs();
// Find the invoke call and mark all of the callee-saved registers as
// 'implicit defined' so that they're spilled. This prevents code from
Modified: llvm/trunk/lib/Target/Hexagon/HexagonEarlyIfConv.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Hexagon/HexagonEarlyIfConv.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Hexagon/HexagonEarlyIfConv.cpp (original)
+++ llvm/trunk/lib/Target/Hexagon/HexagonEarlyIfConv.cpp Sun Dec 13 03:26:17 2015
@@ -939,7 +939,7 @@ void HexagonEarlyIfConversion::removeBlo
B->removeSuccessor(B->succ_begin());
for (auto I = B->pred_begin(), E = B->pred_end(); I != E; ++I)
- (*I)->removeSuccessor(B);
+ (*I)->removeSuccessor(B, true);
Deleted.insert(B);
MDT->eraseNode(B);
@@ -1001,6 +1001,7 @@ void HexagonEarlyIfConversion::mergeBloc
MachineBasicBlock::succ_iterator I, E = SuccB->succ_end();
for (I = SuccB->succ_begin(); I != E; ++I)
PredB->addSuccessor(*I);
+ PredB->normalizeSuccProbs();
replacePhiEdges(SuccB, PredB);
removeBlock(SuccB);
if (!TermOk)
Modified: llvm/trunk/lib/Target/Mips/MipsLongBranch.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Mips/MipsLongBranch.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Mips/MipsLongBranch.cpp (original)
+++ llvm/trunk/lib/Target/Mips/MipsLongBranch.cpp Sun Dec 13 03:26:17 2015
@@ -148,7 +148,7 @@ void MipsLongBranch::splitMBB(MachineBas
// Insert NewMBB and fix control flow.
MachineBasicBlock *Tgt = getTargetMBB(*FirstBr);
NewMBB->transferSuccessors(MBB);
- NewMBB->removeSuccessor(Tgt);
+ NewMBB->removeSuccessor(Tgt, true);
MBB->addSuccessor(NewMBB);
MBB->addSuccessor(Tgt);
MF->insert(std::next(MachineFunction::iterator(MBB)), NewMBB);
Modified: llvm/trunk/lib/Target/PowerPC/PPCEarlyReturn.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCEarlyReturn.cpp?rev=255455&r1=255454&r2=255455&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCEarlyReturn.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCEarlyReturn.cpp Sun Dec 13 03:26:17 2015
@@ -153,7 +153,7 @@ protected:
}
for (unsigned i = 0, ie = PredToRemove.size(); i != ie; ++i)
- PredToRemove[i]->removeSuccessor(&ReturnMBB);
+ PredToRemove[i]->removeSuccessor(&ReturnMBB, true);
if (Changed && !ReturnMBB.hasAddressTaken()) {
// We now might be able to merge this blr-only block into its
@@ -163,7 +163,7 @@ protected:
if (PrevMBB.isLayoutSuccessor(&ReturnMBB) && PrevMBB.canFallThrough()) {
// Move the blr into the preceding block.
PrevMBB.splice(PrevMBB.end(), &ReturnMBB, I);
- PrevMBB.removeSuccessor(&ReturnMBB);
+ PrevMBB.removeSuccessor(&ReturnMBB, true);
}
}
More information about the llvm-commits
mailing list