[llvm] 21504f4 - [CodeGenPrepare] Maintain BranchProbabilityInfo up-to-date in bypassSlowDivision (#212058)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 10:14:35 PDT 2026
Author: Ralender
Date: 2026-08-18T19:14:30+02:00
New Revision: 21504f4ea7f17d038ab528256a5c73f3db2922ff
URL: https://github.com/llvm/llvm-project/commit/21504f4ea7f17d038ab528256a5c73f3db2922ff
DIFF: https://github.com/llvm/llvm-project/commit/21504f4ea7f17d038ab528256a5c73f3db2922ff.diff
LOG: [CodeGenPrepare] Maintain BranchProbabilityInfo up-to-date in bypassSlowDivision (#212058)
The test case crashes on the release/23.x. What happens is the
`BranchProbabilityInfo` becomes outdated because of a change of CFG in
`bypassSlowDivision` but no update to `BranchProbabilityInfo`.
So when it arrives to `SplitIndirectBrCriticalEdges` and hit the assert
in `setEdgeProbability`, because the Branch probabilities no-longer make
sense.
Added:
llvm/test/Transforms/CodeGenPrepare/X86/bypass-slow-division-bpi-update.ll
Modified:
llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h b/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h
index 981702e6c6ebb..7b4b728dc6a8d 100644
--- a/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h
+++ b/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h
@@ -25,6 +25,7 @@
namespace llvm {
class BasicBlock;
+class BranchProbabilityInfo;
class DomTreeUpdater;
class LoopInfo;
class Value;
@@ -63,7 +64,8 @@ template <> struct DenseMapInfo<DivRemMapKey> {
LLVM_ABI bool
bypassSlowDivision(BasicBlock *BB,
const DenseMap<unsigned int, unsigned int> &BypassWidth,
- DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr);
+ DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
+ BranchProbabilityInfo *BPI = nullptr);
} // end namespace llvm
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 5903c97bfbd6a..7b739df127120 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -620,7 +620,7 @@ bool CodeGenPrepare::_run(Function &F) {
// optimization to those blocks.
BasicBlock *Next = BB->getNextNode();
if (!llvm::shouldOptimizeForSize(BB, PSI, BFI))
- EverMadeChange |= bypassSlowDivision(BB, BypassWidths, DTU, LI);
+ EverMadeChange |= bypassSlowDivision(BB, BypassWidths, DTU, LI, BPI);
BB = Next;
}
}
diff --git a/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp b/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
index ff5bf52b560cf..e51087fa6e4ba 100644
--- a/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
+++ b/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
@@ -18,6 +18,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ValueTracking.h"
@@ -81,7 +82,9 @@ class FastDivInsertionTask {
BasicBlock *MainBB = nullptr;
DomTreeUpdater *DTU = nullptr;
LoopInfo *LI = nullptr;
+ BranchProbabilityInfo *BPI = nullptr;
+ BasicBlock *splitMainBB();
bool isHashLikeValue(Value *V, VisitedSetTy &Visited);
ValueRange getValueRange(Value *Op, VisitedSetTy &Visited);
QuotRemWithBB createSlowBB(BasicBlock *Successor);
@@ -105,7 +108,8 @@ class FastDivInsertionTask {
public:
FastDivInsertionTask(Instruction *I, const BypassWidthsTy &BypassWidths,
- DomTreeUpdater *DTU, LoopInfo *LI);
+ DomTreeUpdater *DTU, LoopInfo *LI,
+ BranchProbabilityInfo *BPI);
Value *getReplacement(DivCacheTy &Cache);
};
@@ -114,8 +118,9 @@ class FastDivInsertionTask {
FastDivInsertionTask::FastDivInsertionTask(Instruction *I,
const BypassWidthsTy &BypassWidths,
- DomTreeUpdater *DTU, LoopInfo *LI)
- : DTU(DTU), LI(LI) {
+ DomTreeUpdater *DTU, LoopInfo *LI,
+ BranchProbabilityInfo *BPI)
+ : DTU(DTU), LI(LI), BPI(BPI) {
switch (I->getOpcode()) {
case Instruction::UDiv:
case Instruction::SDiv:
@@ -261,6 +266,24 @@ ValueRange FastDivInsertionTask::getValueRange(Value *V,
return VALRNG_UNKNOWN;
}
+// Split MainBB and keep BPI up-to-date if its present.
+BasicBlock *FastDivInsertionTask::splitMainBB() {
+ SmallVector<BranchProbability, 4> ExitProbs;
+ if (BPI)
+ for (unsigned I = 0, E = MainBB->getTerminator()->getNumSuccessors();
+ I != E; ++I)
+ ExitProbs.push_back(BPI->getEdgeProbability(MainBB, I));
+
+ BasicBlock *SuccessorBB = SplitBlock(MainBB, SlowDivOrRem, DTU, LI);
+ MainBB->back().eraseFromParent();
+
+ if (BPI) {
+ BPI->setEdgeProbability(SuccessorBB, ExitProbs);
+ BPI->eraseBlock(MainBB);
+ }
+ return SuccessorBB;
+}
+
/// Add new basic block for slow div and rem operations and put it before
/// SuccessorBB.
QuotRemWithBB FastDivInsertionTask::createSlowBB(BasicBlock *SuccessorBB) {
@@ -420,9 +443,7 @@ std::optional<QuotRemPair> FastDivInsertionTask::insertFastDivAndRem() {
// lets us entirely avoid a long div.
// Split the basic block before the div/rem.
- BasicBlock *SuccessorBB = SplitBlock(MainBB, SlowDivOrRem, DTU, LI);
- // Remove the unconditional branch from MainBB to SuccessorBB.
- MainBB->back().eraseFromParent();
+ BasicBlock *SuccessorBB = splitMainBB();
QuotRemWithBB Long;
Long.BB = MainBB;
Long.Quotient = ConstantInt::get(getSlowType(), 0);
@@ -447,9 +468,7 @@ std::optional<QuotRemPair> FastDivInsertionTask::insertFastDivAndRem() {
// them at runtime.
// Split the basic block before the div/rem.
- BasicBlock *SuccessorBB = SplitBlock(MainBB, SlowDivOrRem, DTU, LI);
- // Remove the unconditional branch from MainBB to SuccessorBB.
- MainBB->back().eraseFromParent();
+ BasicBlock *SuccessorBB = splitMainBB();
QuotRemWithBB Fast = createFastBB(SuccessorBB);
QuotRemWithBB Slow = createSlowBB(SuccessorBB);
QuotRemPair Result = createDivRemPhiNodes(Fast, Slow, SuccessorBB);
@@ -475,7 +494,8 @@ std::optional<QuotRemPair> FastDivInsertionTask::insertFastDivAndRem() {
/// profitably bypassed and carried out with a shorter, faster divide.
bool llvm::bypassSlowDivision(BasicBlock *BB,
const BypassWidthsTy &BypassWidths,
- DomTreeUpdater *DTU, LoopInfo *LI) {
+ DomTreeUpdater *DTU, LoopInfo *LI,
+ BranchProbabilityInfo *BPI) {
DivCacheTy PerBBDivCache;
bool MadeChange = false;
@@ -490,7 +510,7 @@ bool llvm::bypassSlowDivision(BasicBlock *BB,
if (I->use_empty())
continue;
- FastDivInsertionTask Task(I, BypassWidths, DTU, LI);
+ FastDivInsertionTask Task(I, BypassWidths, DTU, LI, BPI);
if (Value *Replacement = Task.getReplacement(PerBBDivCache)) {
I->replaceAllUsesWith(Replacement);
I->eraseFromParent();
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/bypass-slow-division-bpi-update.ll b/llvm/test/Transforms/CodeGenPrepare/X86/bypass-slow-division-bpi-update.ll
new file mode 100644
index 0000000000000..054330521788a
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/bypass-slow-division-bpi-update.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -codegen-opt-level=2 -mcpu=generic -mtriple=x86_64 -S < %s | FileCheck %s
+
+define ptr @f(i64 %d) {
+; CHECK-LABEL: define ptr @f(
+; CHECK-SAME: i64 [[D:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: store volatile i64 0, ptr null, align 8
+; CHECK-NEXT: ret ptr null
+;
+entry:
+ br label %header
+
+header:
+ %p = phi i64 [ 0, %latch ], [ 0, %entry ]
+ store volatile i64 %p, ptr null, align 8
+ %rem = srem i64 0, %d
+ switch i32 0, label %dispatch [
+ i32 2, label %dispatch
+ i32 0, label %exit
+ ]
+
+dispatch:
+ indirectbr ptr null, [label %latch]
+
+exit:
+ ret ptr null
+
+latch:
+ %gep = getelementptr [48 x i8], ptr null, i64 %rem
+ br label %header
+}
More information about the llvm-commits
mailing list