[llvm] [EarlyIfConversion] Add cascading conditional branches as a conversion pattern (PR #218415)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 06:52:51 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Jonathan Cohen (jcohen-apple)
<details>
<summary>Changes</summary>
EarlyIfConversion today only calculates the profitability of converting triangle and diamond CFGs into conditional selects. This works well for a single conditional branch, but in the case of cascading conditional branches which are unpredictable, the current heuristic underestimates the cost of misprediction. In the case of cascading conditional branches we may mispredict multiple times along the cascade, and the cost of these re-steers of the frontend can compound, as we have seen in internal workloads.
This addition to EarlyIfConversion utilizes the previously added data dependent analysis for conditional branches to check if we have a cascade of likely-to-be unpredictable branches, and if so, raise the acceptable critical path extension as a function of the cascade depth. We also add an additional hidden parameter to configure the average mispredict likelihood, which we set experimentally at 25%, but may need additional tuning.
---
Patch is 142.23 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218415.diff
4 Files Affected:
- (modified) llvm/lib/CodeGen/EarlyIfConversion.cpp (+583-75)
- (added) llvm/test/CodeGen/AArch64/early-ifcvt-cascade.mir (+1377)
- (modified) llvm/test/CodeGen/AArch64/early-ifcvt-load-to-cond-br-limit.mir (+577)
- (modified) llvm/test/CodeGen/AArch64/early-ifcvt-remarks.mir (+256)
``````````diff
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index bf2664fd4225c..24fc9a90d88cc 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -16,11 +16,16 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/EarlyIfConversion.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SparseSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -65,6 +70,16 @@ static cl::opt<bool> EnableDataDependentBranchAnalysis(
"enable-early-ifcvt-data-dependent", cl::Hidden, cl::init(false),
cl::desc("Enable hard-to-predict branch analysis for if-conversion"));
+// Enable recognition of cascade if-else patterns (chains of triangles).
+static cl::opt<bool> EnableCascadeIfConv(
+ "enable-early-ifcvt-cascade", cl::Hidden, cl::init(false),
+ cl::desc("Enable recognition of cascade if-else patterns"));
+
+// Maximum depth for cascade patterns.
+static cl::opt<unsigned>
+ MaxCascadeDepth("early-ifcvt-max-cascade-depth", cl::Hidden, cl::init(8),
+ cl::desc("Maximum cascade depth for early if-conversion"));
+
// Limit the number steps we take when searching conditions that depend on
// values recently loaded from memory.
static cl::opt<unsigned>
@@ -86,6 +101,8 @@ STATISTIC(NumTrianglesConv, "Number of triangles converted");
STATISTIC(NumDataDependant,
"Number of data dependent conditional branches encountered");
STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
+STATISTIC(NumCascadesSeen, "Number of cascade patterns detected");
+STATISTIC(NumCascadesConv, "Number of cascade patterns converted");
//===----------------------------------------------------------------------===//
// SSAIfConv
@@ -109,6 +126,20 @@ STATISTIC(NumLikelyBiased, "Number of branches with a hot path encountered");
// Head block, and phis in the Tail block are converted to select instructions.
//
namespace {
+
+/// What one cascade collapse step would do to one Tail phi.
+struct CascadeSelectInfo {
+ int CondCycles = 0;
+ bool NeedsSelect = false;
+};
+
+struct CascadeResult {
+ MachineBasicBlock *Head = nullptr;
+ SmallVector<MachineBasicBlock *> Blocks;
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> Selects;
+ explicit operator bool() const { return Head != nullptr; }
+};
+
class SSAIfConv {
const TargetInstrInfo *TII;
const TargetRegisterInfo *TRI;
@@ -213,12 +244,36 @@ class SSAIfConv {
/// initialize the internal state, and return true.
/// If predicate is set try to predicate the block otherwise try to
/// speculatively execute it.
- bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false);
+ bool canConvertIf(MachineBasicBlock *MBB, bool Predicate = false,
+ bool AllowMultiSuccTBB = false,
+ MachineBasicBlock *ExplicitTail = nullptr);
/// convertIf - If-convert the last block passed to canConvertIf(), assuming
/// it is possible. Add any blocks that are to be erased to RemoveBlocks.
void convertIf(SmallVectorImpl<MachineBasicBlock *> &RemoveBlocks,
bool Predicate = false);
+
+ /// matchCascade - match a cascade of conditional branches by walking up
+ /// from MBB as a potential cascade end. A cascade is a chain of triangles
+ /// where each block has 2 successors (next cascade block + Tail) and the
+ /// last block has 1 successor (Tail). Returns the discovered Head and list
+ /// of cascade blocks [BB1, ..., BBn] if valid, empty result otherwise.
+ ///
+ /// Head --------+
+ /// | |
+ /// v |
+ /// BB1 ----------+
+ /// | |
+ /// v |
+ /// BB2 ----------+
+ /// | |
+ /// ...
+ /// ...
+ /// | |
+ /// v v
+ /// BBn (MBB) -> Tail
+ ///
+ CascadeResult matchCascade(MachineBasicBlock *MBB);
};
} // end anonymous namespace
@@ -469,7 +524,9 @@ bool SSAIfConv::findInsertionPoint() {
/// canConvertIf - analyze the sub-cfg rooted in MBB, and return true if it is
/// a potential candidate for if-conversion. Fill out the internal state.
///
-bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
+bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate,
+ bool AllowMultiSuccTBB,
+ MachineBasicBlock *ExplicitTail) {
Head = MBB;
TBB = FBB = Tail = nullptr;
@@ -482,31 +539,40 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (Succ0->pred_size() != 1)
std::swap(Succ0, Succ1);
- if (Succ0->pred_size() != 1 || Succ0->succ_size() != 1)
+ if (Succ0->pred_size() != 1 ||
+ (!AllowMultiSuccTBB && Succ0->succ_size() != 1))
return false;
- Tail = Succ0->succ_begin()[0];
-
- // This is not a triangle.
- if (Tail != Succ1) {
- // Check for a diamond. We won't deal with any critical edges.
- if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
- Succ1->succ_begin()[0] != Tail)
- return false;
- LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << "/"
- << printMBBReference(*Succ1) << " -> "
+ // Use explicit tail if provided (for cascade validation), otherwise compute.
+ if (ExplicitTail) {
+ Tail = ExplicitTail;
+ LLVM_DEBUG(dbgs() << "\nCascade triangle: " << printMBBReference(*Head)
+ << " -> " << printMBBReference(*Succ0) << " -> "
<< printMBBReference(*Tail) << '\n');
-
- // Live-in physregs are tricky to get right when speculating code.
- if (!Tail->livein_empty()) {
- LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
- return false;
- }
} else {
- LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
- << printMBBReference(*Succ0) << " -> "
- << printMBBReference(*Tail) << '\n');
+ Tail = Succ0->succ_begin()[0];
+
+ // This is not a triangle.
+ if (Tail != Succ1) {
+ // Check for a diamond. We won't deal with any critical edges.
+ if (Succ1->pred_size() != 1 || Succ1->succ_size() != 1 ||
+ Succ1->succ_begin()[0] != Tail)
+ return false;
+ LLVM_DEBUG(dbgs() << "\nDiamond: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << "/"
+ << printMBBReference(*Succ1) << " -> "
+ << printMBBReference(*Tail) << '\n');
+
+ // Live-in physregs are tricky to get right when speculating code.
+ if (!Tail->livein_empty()) {
+ LLVM_DEBUG(dbgs() << "Tail has live-ins.\n");
+ return false;
+ }
+ } else {
+ LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
+ << printMBBReference(*Succ0) << " -> "
+ << printMBBReference(*Tail) << '\n');
+ }
}
// This is a triangle or a diamond.
@@ -588,10 +654,15 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB, bool Predicate) {
if (!findInsertionPoint())
return false;
- if (isTriangle())
- ++NumTrianglesSeen;
- else
- ++NumDiamondsSeen;
+ // ExplicitTail means this is a validation call. Don't count towards the
+ // triangle / diamonds seen stat - that should happen only when we consider
+ // them for conversion seperately.
+ if (!ExplicitTail) {
+ if (isTriangle())
+ ++NumTrianglesSeen;
+ else
+ ++NumDiamondsSeen;
+ }
return true;
}
@@ -641,6 +712,121 @@ static bool hasSameValue(const MachineRegisterInfo &MRI,
return TIdx == FIdx;
}
+/// matchCascade - match a cascade pattern by walking up from MBB.
+/// MBB is a potential cascade end (last block before Tail).
+/// A cascade is a chain of triangles where:
+/// - Head has 2 successors: one to first cascade block, one to Tail
+/// - Each cascade block has 2 successors: next cascade + Tail (or 1 for last)
+/// - Last cascade block (MBB) has 1 successor: Tail
+/// Returns the discovered Head and cascade blocks if valid, empty otherwise.
+///
+/// Head --------+
+/// | |
+/// v |
+/// BB1 ----------+
+/// | |
+/// v |
+/// BB2 ----------+
+/// | |
+/// ...
+/// ...
+/// | |
+/// v v
+/// BBn (MBB) -> Tail
+///
+CascadeResult SSAIfConv::matchCascade(MachineBasicBlock *MBB) {
+ // Cascade end must have exactly 1 successor (to Tail) and 1 predecessor
+ if (MBB->succ_size() != 1 || MBB->pred_size() != 1)
+ return {};
+
+ MachineBasicBlock *Tail = MBB->succ_begin()[0];
+
+ // Collect cascade blocks from bottom to top: [BBn, BB(n-1), ..., BB1, Head]
+ SmallVector<MachineBasicBlock *> Blocks;
+ Blocks.push_back(MBB);
+ MachineBasicBlock *Current = MBB;
+
+ // Walk up the cascade chain
+ while (true) {
+ MachineBasicBlock *Pred = Current->pred_begin()[0];
+
+ // Predecessor must have 2 successors: Current and Tail
+ if (Pred->succ_size() != 2)
+ break;
+
+ MachineBasicBlock *S0 = Pred->succ_begin()[0];
+ MachineBasicBlock *S1 = Pred->succ_begin()[1];
+
+ // One successor must be Current, the other must be Tail
+ if (!((S0 == Current && S1 == Tail) || (S1 == Current && S0 == Tail)))
+ break;
+
+ Blocks.push_back(Pred);
+
+ // If Pred has multiple predecessors, it's the Head - stop here
+ if (Pred->pred_size() != 1)
+ break;
+
+ // Pred has 1 predecessor, so the cascade continues. If we've reached
+ // max depth, bail out entirely - we don't want to convert a partial cascade.
+ if (Blocks.size() > MaxCascadeDepth) {
+ LLVM_DEBUG(dbgs() << "Cascade extends beyond max depth "
+ << MaxCascadeDepth << ", not converting.\n");
+ return {};
+ }
+
+ Current = Pred;
+ }
+
+ // Need at least 3 blocks: [BBn, BB1, Head] for a minimal 2-block cascade
+ if (Blocks.size() < 3)
+ return {};
+
+ // Reverse to get [HEAD, BB1, BB2, ..., BBn] order
+ std::reverse(Blocks.begin(), Blocks.end());
+
+ // Check that we can actually convert all the blocks in the cascade.
+ // We skip the last cascade block (BBn) since it has a single successor
+ // (to Tail), and is only used as a TBB (not a HEAD) - its speculatability
+ // is validated when we check its predecessor.
+ SmallVector<SmallVector<CascadeSelectInfo, 4>> CollectedSelects;
+ size_t NumBlocksToValidate = Blocks.size() - 1;
+ for (size_t I = 0; I < NumBlocksToValidate; ++I) {
+ if (!canConvertIf(Blocks[I], /*Predicate=*/false,
+ /*AllowMultiSuccTBB=*/true,
+ /*ExplicitTail=*/Tail)) {
+ LLVM_DEBUG(dbgs() << "Cannot convert cascade, block "
+ << printMBBReference(*Blocks[I])
+ << " is not if-convertible.\n");
+ return {};
+ }
+ assert((CollectedSelects.empty() ||
+ CollectedSelects.back().size() == PHIs.size()) &&
+ "Tail phi list changed between cascade blocks");
+
+ // Capture PHI info from canConvertIf's canInsertSelect call.
+ auto &StepSelects = CollectedSelects.emplace_back();
+ for (const PHIInfo &PI : PHIs)
+ StepSelects.push_back(
+ {PI.CondCycles, !hasSameValue(*MRI, TII, PI.TReg, PI.FReg)});
+ }
+
+ // The first block in Blocks is the cascade Head
+ MachineBasicBlock *Head = Blocks.front();
+
+ // Remove Head from the cascade blocks to convert
+ Blocks.erase(Blocks.begin());
+
+ LLVM_DEBUG({
+ dbgs() << "\nCascade found: " << printMBBReference(*Head) << " -> [";
+ for (auto *BB : Blocks)
+ dbgs() << printMBBReference(*BB) << ", ";
+ dbgs() << "] -> " << printMBBReference(*Tail) << "\n";
+ });
+
+ return {Head, std::move(Blocks), std::move(CollectedSelects)};
+}
+
/// replacePHIInstrs - Completely replace PHI instructions with selects.
/// This is possible when the only Tail predecessors are the if-converted
/// blocks.
@@ -843,6 +1029,10 @@ class EarlyIfConverter {
/// each block to the number of instructions scanned in it.
DenseMap<const MachineBasicBlock *, unsigned> NoCallBlocksCache;
+ /// Set of blocks that must be converted (part of a cascade).
+ /// These blocks bypass normal profitability checks in shouldConvertIf().
+ SmallPtrSet<MachineBasicBlock *, 16> MustConvertBlocks;
+
public:
EarlyIfConverter(MachineDominatorTree &DT, MachineLoopInfo &LI,
MachineTraceMetrics &MTM, MachineBranchProbabilityInfo *MBPI)
@@ -853,11 +1043,20 @@ class EarlyIfConverter {
private:
bool tryConvertIf(MachineBasicBlock *);
+ void detectCascades(MachineBasicBlock *);
+ void convertIf();
void invalidateTraces();
bool shouldConvertIf();
- bool isConditionDataDependent();
- bool doOperandsComeFromMemory(const MachineInstr *ConditionDef);
+ bool isConditionDataDependent(MachineBasicBlock *BB, bool RecordStats = true);
+ bool isCascadeDataDependent(MachineBasicBlock *Head,
+ ArrayRef<MachineBasicBlock *> CascadeBlocks);
+ bool doOperandsComeFromMemory(const MachineInstr *ConditionDef,
+ MachineBasicBlock *BB);
bool hasCallOrLoopInRange(const MachineInstr *From, const MachineInstr *To);
+ bool shouldConvertCascade(CascadeResult &Cascade, MachineBasicBlock *Tail);
+ bool hasEnoughILP(MachineBasicBlock *TraceBlock,
+ SmallVectorImpl<MachineBasicBlock *> &ExtraBlocks,
+ unsigned CritLimit);
};
class EarlyIfConverterLegacy : public MachineFunctionPass {
@@ -1047,9 +1246,10 @@ bool EarlyIfConverter::hasCallOrLoopInRange(const MachineInstr *From,
/// depend on values loaded from memory (unless they are loop invariant,
/// or come from a constant pool). The walk starts from the definition of
/// ConditionDef's first operand, which is not ConditionDef itself for
-/// instructions such as FCMPSrr, where that operand is a use.
+/// instructions such as FCMPSrr, where that operand is a use. BB is the block
+/// whose terminator consumes the condition.
bool EarlyIfConverter::doOperandsComeFromMemory(
- const MachineInstr *ConditionDef) {
+ const MachineInstr *ConditionDef, MachineBasicBlock *BB) {
Register Reg = ConditionDef->getOperand(0).getReg();
if (!Reg.isVirtual())
return false;
@@ -1057,10 +1257,10 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
LLVM_DEBUG(dbgs() << " doOperandsComeFromMemory starting from reg "
<< printReg(Reg) << "\n");
- // The condition is consumed by the branch terminating Head, so this is the
+ // The condition is consumed by the branch terminating BB, so this is the
// end of the interval a load has to survive without a call in between.
- const MachineInstr *Br = &*IfConv.Head->getFirstTerminator();
- MachineLoop *IfConvLoop = Loops->getLoopFor(IfConv.Head);
+ const MachineInstr *Br = &*BB->getFirstTerminator();
+ MachineLoop *IfConvLoop = Loops->getLoopFor(BB);
// Walk the def-use chain.
SmallPtrSet<const MachineInstr *, 8> VisitedInstrs;
@@ -1127,9 +1327,12 @@ bool EarlyIfConverter::doOperandsComeFromMemory(
}
/// Check if the branch condition is data-dependent (comes from memory loads).
-bool EarlyIfConverter::isConditionDataDependent() {
+/// RecordStats should be false when the same branch may be examined again
+/// later, so that it is only counted once.
+bool EarlyIfConverter::isConditionDataDependent(MachineBasicBlock *BB,
+ bool RecordStats) {
TargetInstrInfo::MachineBranchPredicate MBP;
- if (TII->analyzeBranchPredicate(*IfConv.Head, MBP, /*AllowModify=*/false))
+ if (TII->analyzeBranchPredicate(*BB, MBP, /*AllowModify=*/false))
return false;
if (!MBP.ConditionDef)
@@ -1138,29 +1341,52 @@ bool EarlyIfConverter::isConditionDataDependent() {
// If the branch is biased (not 50/50), don't consider it data dependent.
// This is to prevent converting unprofitable checks such as
// `x[i] != 0;`
- auto TBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.TBB);
- auto FBBProb = MBPI->getEdgeProbability(IfConv.Head, IfConv.FBB);
- if (TBBProb != FBBProb) {
- ++NumLikelyBiased;
- return false;
+ if (MBP.TrueDest && MBP.FalseDest && MBPI) {
+ auto TBBProb = MBPI->getEdgeProbability(BB, MBP.TrueDest);
+ auto FBBProb = MBPI->getEdgeProbability(BB, MBP.FalseDest);
+ if (TBBProb != FBBProb) {
+ if (RecordStats)
+ ++NumLikelyBiased;
+ return false;
+ }
}
// Check if operands used to compute the branch condition were loaded recently
// from memory, starting by the ConditionDef itself and walking up the use-def
// chain.
- if (doOperandsComeFromMemory(MBP.ConditionDef)) {
- ++NumDataDependant;
+ if (doOperandsComeFromMemory(MBP.ConditionDef, BB)) {
+ if (RecordStats)
+ ++NumDataDependant;
return true;
}
return false;
}
-// Adjust cycles with downward saturation.
-static unsigned adjCycles(unsigned Cyc, int Delta) {
- if (Delta < 0 && Cyc + Delta > Cyc)
- return 0;
- return Cyc + Delta;
+/// Check if ALL branches in a cascade are data-dependent (come from loads).
+/// Cascade conversion always requires data-dependent branches, so this check
+/// is enabled whenever cascade conversion is enabled.
+bool EarlyIfConverter::isCascadeDataDependent(
+ MachineBasicBlock *Head, ArrayRef<MachineBasicBlock *> CascadeBlocks) {
+
+ // Check Head block
+ if (!isConditionDataDependent(Head, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: Head not data-dependent\n");
+ return false;
+ }
+
+ // Check all cascade blocks except the last one (which has only 1 successor).
+ // Don't record stats because we are not converting anything yet.
+ for (auto *CascadeBlock : CascadeBlocks.drop_back()) {
+ if (!isConditionDataDependent(CascadeBlock, /*RecordStats=*/false)) {
+ LLVM_DEBUG(dbgs() << "Cascade: " << printMBBReference(*CascadeBlock)
+ << " not data-dependent\n");
+ return false;
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "Cascade: all branches are data-dependent\n");
+ return true;
}
namespace {
@@ -1168,20 +1394,251 @@ namespace {
struct Cycles {
const char *Key;
unsigned Value;
+ Cycles(const char *K, unsigned V) : Key(K), Value(V) {}
};
template <typename Remark> Remark &operator<<(Remark &R, Cycles C) {
return R << ore::NV(C.Key, C.Value) << (C.Value == 1 ? " cycle" : " cycles");
}
} // anonymous namespace
+// Adjust cycles with downward saturation.
+static unsigned adjCycles(unsigned Cyc, int Delta) {
+ if (Delta < 0 && Cyc + Delta > Cyc)
+ return 0;
+ return Cyc + Delta;
+}
+
+/// Count the instructions of MBB that a predecessor would have to speculate to
+/// absorb it. This matches what canSpeculateInstrs() checks against
+/// BlockInstrLimit: the non-debug instructions ahead of the terminators.
+/// If we don't perform this check we could bail out of a cascade conversion
+/// midway by exceeding `BlockInstrLimit`.
+static unsigned countSpeculatedInstrs(const MachineBasicBlock &MBB) {
+ return count_if(make_range(MBB.begin(), MBB.getFirstTerminator()),
+ [](const MachineInstr &MI) { return !MI.isDebugInstr(); });
+}
+
+/// Apply profitability check for cascade conversion.
+bool EarlyIfConverter::shouldConvertCascade(CascadeResult &Cascade,
+ MachineBasicBlock *Tail) {
+ MachineBasicBlock *Head = Cascade.Head;
+ auto &CascadeBlocks = Cascade.Blocks;
+
+ if (!isCascadeDataDependent(Head, CascadeBlocks)) {
+ LLVM_DEBUG(dbgs() << "Cascade: not all branches are data-dependent\n");
+ return false;
+ }
+
+ // Calculate CritLimit using cascade formula.
+ unsigned CascadeSize = CascadeBlocks.size();
+ unsigned MispredictPenalty = STI->getMispredictionPenalty();
+ unsigned CritLimit =
+ std::min(Mispredict...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218415
More information about the llvm-commits
mailing list