[llvm] [CycleInfo] Identify cycles with a single-pass DFS algorithm (PR #210491)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 20:06:20 PDT 2026
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/210491
>From 0c235be238813d7f5dc925ced840e1dd0a5e9c5b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Fri, 17 Jul 2026 09:43:39 -0700
Subject: [PATCH 1/6] [CycleInfo] Identify cycles with a single-pass DFS
algorithm
Replace the Havlak-Tarjan construction in GenericCycleInfoCompute, a DFS
followed by a reverse-preorder scan, with the single-pass algorithm of
Wei, Mao, Zou and Chen, "A New Algorithm for Identifying Loops in
Decompilation" (SAS 2007). One depth-first traversal tags every block
with its innermost loop header on the fly; tagLoopHeader weaves the
per-block header chains, replacing UNION-FIND.
The flat forest is reconstructed from the tags, dropping the temporary
cycle objects and the per-block worklist passes. An edge re-entering an
already-closed cycle records non-header entries, so entries need no
predecessor scan.
The cycle sets, headers, reducibility and nesting are identical for the
given DFS order, cross-checked against the old construction on random
reducible and irreducible CFGs. Two implementation-defined orders change
(with minor test churn): sibling cycles are laid out in decreasing
header preorder, and non-header entries in block preorder.
Construction cost (51x require+invalidate of <cycles> minus a 1x run,
median of interleaved rounds, x86-64): sqlite3.bc executes 14% fewer
instructions and 21% fewer cycles; a 1500-deep loop nest executes 45%
fewer instructions and 44% fewer cycles.
Aided by Claude Fable 5
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 453 ++++++++++-------------
llvm/include/llvm/ADT/GenericCycleInfo.h | 36 +-
llvm/test/Analysis/CycleInfo/basic.ll | 2 +-
llvm/test/CodeGen/X86/cycle-info.mir | 6 +-
4 files changed, 212 insertions(+), 285 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index e2da17af20826..dc45ee19bb8a4 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -26,8 +26,8 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/GenericCycleInfo.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
-#include <deque>
#include <iterator>
#define DEBUG_TYPE "generic-cycle-impl"
@@ -207,7 +207,7 @@ void GenericCycleInfo<ContextT>::verifyCycleNest(CycleRef C) const {
// Check the parent cycle.
if (Cyc.hasParent()) {
- assert(is_contained(children(CycleRef(Cyc.ParentIndex)), C) &&
+ assert(is_contained(children(Cyc.Parent), C) &&
"Cycle is not a subcycle of its parent!");
}
#endif
@@ -222,90 +222,84 @@ template <typename ContextT> class GenericCycleInfoCompute {
CycleInfoT &Info;
- struct DFSInfo {
- unsigned Start = 0; // DFS start; positive if block is found
- unsigned End = 0; // DFS end
-
- DFSInfo() = default;
- explicit DFSInfo(unsigned Start) : Start(Start) {}
-
- explicit operator bool() const { return Start; }
-
- /// Whether this node is an ancestor (or equal to) the node \p Other
- /// in the DFS tree.
- bool isAncestorOf(const DFSInfo &Other) const {
- return Start <= Other.Start && Other.End <= End;
- }
+ /// Sentinel header-preorder rank meaning "no cycle".
+ static constexpr unsigned NoCycle = ~0u;
+ /// Sentinel block number meaning "no block".
+ static constexpr unsigned NoBlock = ~0u;
+
+ /// Per-block state indexed by block number.
+ struct BlockInfo {
+ /// The block this entry describes; null for an unreachable block.
+ BlockT *Block = nullptr;
+ /// Block number of the innermost loop header found so far; NoBlock if none.
+ unsigned LoopHeader = NoBlock;
+ /// 1-based position on the current DFS path; 0 once the block leaves it.
+ unsigned DFSPPos = 0;
+ /// Header-preorder rank of the innermost cycle containing this block (the
+ /// cycle it heads if IsHeader); NoCycle if none, or unreachable.
+ unsigned CycleIdx = NoCycle;
+ bool IsHeader = false;
};
-
- // Indexed by block number.
- SmallVector<DFSInfo, 8> BlockDFSInfo;
- SmallVector<BlockT *, 8> BlockPreorder;
-
- /// Append-only cycles discovered so far, in creation order.
- std::deque<CycleT> AllCycles;
-
- /// Flat log of child attachments, in attach order. Cycles are only ever
- /// attached to the newest cycle, so each cycle's children occupy the
- /// contiguous slice [IdxBegin, Depth) of this log.
- SmallVector<CycleT *, 8> AttachedChildren;
-
- SmallVector<CycleT *, 8> TopLevelCycles;
+ SmallVector<BlockInfo, 8> BlockInfos;
+ /// Reachable block numbers in DFS preorder.
+ SmallVector<unsigned, 8> Preorder;
+ /// Records (block B, header H): an edge from outside re-enters the closed
+ /// cycle headed by H at B, making B a non-header entry of it.
+ SmallVector<std::pair<unsigned, unsigned>, 8> Reentries;
GenericCycleInfoCompute(const GenericCycleInfoCompute &) = delete;
GenericCycleInfoCompute &operator=(const GenericCycleInfoCompute &) = delete;
- DFSInfo getDFSInfo(BlockT *B) const {
- unsigned Number = GraphTraits<BlockT *>::getNumber(B);
- return BlockDFSInfo[Number];
+ static unsigned num(const BlockT *B) {
+ return GraphTraits<const BlockT *>::getNumber(B);
}
- DFSInfo &getOrInsertDFSInfo(BlockT *B) {
- unsigned Number = GraphTraits<BlockT *>::getNumber(B);
- return BlockDFSInfo[Number];
- }
-
- /// Make top-level cycle \p Child a child of \p NewParent.
- void moveTopLevelCycleToNewParent(CycleT *NewParent, CycleT *Child) {
- assert((!Child->hasParent() && !NewParent->hasParent()) &&
- "NewParent and Child must be both top level cycle!\n");
- auto Pos = llvm::find(TopLevelCycles, Child);
- assert(Pos != TopLevelCycles.end());
- *Pos = TopLevelCycles.back();
- TopLevelCycles.pop_back();
- AttachedChildren.push_back(Child);
- // NewParent is the newest cycle and its creation-order index is
- // AllCycles.size() - 1.
- assert(NewParent == &AllCycles.back() &&
- "attach slices in AttachedChildren must stay contiguous");
- Child->ParentIndex = AllCycles.size() - 1;
+ BlockInfo &info(unsigned Number) { return BlockInfos[Number]; }
+
+ /// Weave loop header \p H (and its own header chain) into the loop header
+ /// chain of \p B, keeping the chain ordered from innermost to outermost by
+ /// DFS-path position. Building this chain on the fly is why the algorithm
+ /// needs no union-find (used in the Havlak algorithm) at all.
+ void tagLoopHeader(unsigned B, unsigned H) {
+ if (H == NoBlock)
+ return;
+ // Invariant: info(B).DFSPPos >= info(H).DFSPPos.
+ while (B != H) {
+ unsigned IH = info(B).LoopHeader;
+ if (IH == NoBlock) {
+ // B's chain ended: append the rest of H's chain.
+ info(B).LoopHeader = H;
+ return;
+ }
+ // Keep whichever candidate header is inner (larger DFS-path position).
+ if (info(IH).DFSPPos >= info(H).DFSPPos)
+ B = IH;
+ else {
+ info(B).LoopHeader = H;
+ B = H;
+ H = IH;
+ }
+ }
}
- /// Record that \p Block's innermost cycle is the one currently being built
- /// (always AllCycles.back()), storing its creation-order index. flatten()
- /// remaps it to the preorder index.
- void recordInnermostCycle(BlockT *Block) {
- Info.BlockMap[GraphTraits<BlockT *>::getNumber(Block)] =
- AllCycles.size() - 1;
- }
+ void dfs(BlockT *EntryBlock);
+ void flatten(ArrayRef<BlockT *> Headers, ArrayRef<unsigned> ChildHead,
+ ArrayRef<unsigned> NextSibling, ArrayRef<unsigned> OwnCount,
+ unsigned TopHead);
public:
GenericCycleInfoCompute(CycleInfoT &Info) : Info(Info) {}
void run(FunctionT *F);
-
-private:
- void dfs(FunctionT *F, BlockT *EntryBlock);
- void flatten(ArrayRef<BlockT *> Order);
};
template <typename ContextT>
-void GenericCycleInfo<ContextT>::addToBlockMap(BlockT *Block, CycleT *Cycle) {
- // The caller should ensure that BlockMap is large enough. \p Cycle is a flat
+void GenericCycleInfo<ContextT>::addToBlockMap(BlockT *Block, CycleRef C) {
+ // The caller should ensure that BlockMap is large enough. C is a flat
// cycle, so its preorder index is well-defined.
verifyBlockNumberEpoch(Block->getParent());
unsigned Number = GraphTraits<BlockT *>::getNumber(Block);
- BlockMap[Number] = getCycleIndex(*Cycle);
+ BlockMap[Number] = C;
}
template <typename ContextT>
@@ -315,7 +309,7 @@ void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleRef C) {
unsigned Number = GraphTraits<BlockT *>::getNumber(Block);
if (Number >= BlockMap.size())
BlockMap.resize(GraphTraits<FunctionT *>::getMaxNumber(Block->getParent()),
- NoCycle);
+ CycleRef());
// Insert Block at the end of Cyc's slice and shift every later cycle's
// range right. Ranges straddling Pos belong to Cyc's ancestors and are
@@ -329,82 +323,80 @@ void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleRef C) {
++X.IdxEnd;
}
}
- addToBlockMap(Block, &Cyc);
+ addToBlockMap(Block, C);
// Cyc and its ancestors gain the new block: extend each one's slice and
// invalidate its exit-block cache in a single walk up the tree.
- for (unsigned I = getCycleIndex(Cyc); I != NoCycle;
- I = Cycles[I].ParentIndex) {
- ++Cycles[I].IdxEnd;
+ for (CycleRef I = C; I; I = deref(I).Parent) {
+ ++deref(I).IdxEnd;
if (!ExitBlocksCaches.empty())
- ExitBlocksCaches[I].clear();
+ ExitBlocksCaches[I.Index].clear();
}
}
-/// Move the discovered forest into Info's flat preorder array. Assigns preorder
-/// IDs, depths and descendant counts, remaps BlockMap from creation-order to
-/// preorder indices, and lays out every cycle's blocks in BlockLayout.
+/// Lay the discovered cycle forest out into Info's flat preorder array: number
+/// the cycles in Euler-tour order, set each one's parent, depth and descendant
+/// count, place every block into its innermost cycle's region of BlockLayout,
+/// and fill BlockMap. Arrays are keyed by header-preorder rank.
template <typename ContextT>
-void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<BlockT *> Order) {
- unsigned N = AllCycles.size();
+void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<BlockT *> Headers,
+ ArrayRef<unsigned> ChildHead,
+ ArrayRef<unsigned> NextSibling,
+ ArrayRef<unsigned> OwnCount,
+ unsigned TopHead) {
+ unsigned N = Headers.size();
Info.NumCycles = N;
if (!N)
return;
Info.Cycles = std::make_unique<CycleT[]>(N);
- // Walk the cycle forest as an Euler tour. On entry, a cycle's IdxEnd still
- // holds its own-block count (accumulated during run()); reserve that many
- // slots for its own region [Cursor, Cursor + count). Its children take the
- // following slots, so on leaving, Cursor is the cycle's real range end, which
- // overwrites the now-consumed count in IdxEnd.
+ // Walk the cycle forest as an Euler tour. On entry a cycle reserves [Cursor,
+ // Cursor + OwnCount) for its own blocks (IdxBegin temporarily holds that
+ // region's end; the fill loop below walks it back down); its descendants take
+ // the following slots, so on exit Cursor is its IdxEnd.
+ SmallVector<unsigned, 8> FlatIdx(N);
struct Frame {
- CycleT *Flat;
- unsigned ChildCur, ChildEnd;
- unsigned ID;
+ unsigned Flat;
+ unsigned Child; // Next child to enter, NoCycle once exhausted.
};
SmallVector<Frame, 8> Stack;
- unsigned Cursor = 0;
- unsigned NextID = 0;
- auto enter = [&](CycleT *Temp, CycleT *Parent) {
+ unsigned Cursor = 0, NextID = 0;
+ auto enter = [&](unsigned C, CycleRef Parent) {
unsigned ID = NextID++;
+ FlatIdx[C] = ID;
CycleT &Flat = Info.Cycles[ID];
- Flat.ParentIndex =
- Parent ? Info.getCycleIndex(*Parent) : CycleInfoT::NoCycle;
- Flat.Depth = Parent ? Parent->Depth + 1 : 1;
- Flat.Entries = std::move(Temp->Entries);
- Cursor += Temp->IdxEnd; // IdxEnd currently holds Temp's own-block count.
- Flat.IdxBegin = Cursor; // Real begin restored by the fill loop below.
- Stack.push_back({&Flat, Temp->IdxBegin, Temp->Depth, ID});
- // Temp's IdxBegin now holds the flat index, for the BlockMap remap below.
- Temp->IdxBegin = ID;
+ Flat.Parent = Parent;
+ Flat.Depth = Parent ? Info.deref(Parent).Depth + 1 : 1;
+ Flat.appendEntry(Headers[C]);
+ Cursor += OwnCount[C];
+ Flat.IdxBegin = Cursor;
+ Stack.push_back({ID, ChildHead[C]});
};
- for (CycleT *TLC : TopLevelCycles) {
- enter(TLC, nullptr);
+ for (auto TLC = TopHead; TLC != NoCycle; TLC = NextSibling[TLC]) {
+ enter(TLC, CycleRef());
while (!Stack.empty()) {
Frame &F = Stack.back();
- if (F.ChildCur != F.ChildEnd) {
- enter(AttachedChildren[F.ChildCur++], F.Flat);
+ if (F.Child != NoCycle) {
+ unsigned C = F.Child;
+ F.Child = NextSibling[C];
+ enter(C, CycleRef(F.Flat));
} else {
- F.Flat->IdxEnd = Cursor;
- F.Flat->NumDescendants = NextID - F.ID - 1;
+ CycleT &Flat = Info.Cycles[F.Flat];
+ Flat.IdxEnd = Cursor;
+ Flat.NumDescendants = NextID - F.Flat - 1;
Stack.pop_back();
}
}
}
- // Place every block into its innermost cycle's own region, remapping its
- // BlockMap entry from a creation-order index to the flat preorder index.
+ // Place every block into its innermost cycle's own region.
Info.BlockLayout.resize_for_overwrite(Cursor);
- for (BlockT *B : llvm::reverse(Order)) {
- unsigned Number = GraphTraits<const BlockT *>::getNumber(B);
- unsigned Created = Info.BlockMap[Number];
- if (Created != CycleInfoT::NoCycle) {
- // Created indexes AllCycles; enter() stashed the flat preorder index in
- // that temporary node's IdxBegin.
- unsigned Flat = AllCycles[Created].IdxBegin;
- Info.BlockMap[Number] = Flat;
- CycleT &FlatCycle = Info.Cycles[Flat];
- Info.BlockLayout[--FlatCycle.IdxBegin] = B;
- }
+ for (unsigned N : llvm::reverse(Preorder)) {
+ BlockInfo &BI = info(N);
+ if (BI.CycleIdx == NoCycle)
+ continue;
+ unsigned Flat = FlatIdx[BI.CycleIdx];
+ Info.BlockMap[N] = CycleRef(Flat);
+ Info.BlockLayout[--Info.Cycles[Flat].IdxBegin] = BI.Block;
}
}
@@ -412,174 +404,123 @@ void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<BlockT *> Order) {
template <typename ContextT>
void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
BlockT *EntryBlock = GraphTraits<FunctionT *>::getEntryNode(F);
- LLVM_DEBUG(errs() << "Entry block: " << Info.Context.print(EntryBlock)
- << "\n");
- dfs(F, EntryBlock);
-
- SmallVector<BlockT *, 8> Worklist;
-
- for (BlockT *HeaderCandidate : llvm::reverse(BlockPreorder)) {
- const DFSInfo CandidateInfo = getDFSInfo(HeaderCandidate);
-
- for (BlockT *Pred : predecessors(HeaderCandidate)) {
- const DFSInfo PredDFSInfo = getDFSInfo(Pred);
- // This automatically ignores unreachable predecessors since they have
- // zeros in their DFSInfo.
- if (CandidateInfo.isAncestorOf(PredDFSInfo))
- Worklist.push_back(Pred);
- }
- if (Worklist.empty()) {
- continue;
+ BlockInfos.assign(GraphTraits<FunctionT *>::getMaxNumber(F), BlockInfo{});
+
+ dfs(EntryBlock);
+
+ // Number the cycles by their header's preorder rank and resolve every
+ // block's innermost cycle in one pass: a block's LoopHeader is a DFS
+ // ancestor and so already numbered, and parents get smaller ranks than
+ // their children.
+ SmallVector<BlockT *, 8> Headers;
+ SmallVector<unsigned, 8> ChildHead, NextSibling, OwnCount;
+ unsigned TopHead = NoCycle;
+ for (unsigned N : Preorder) {
+ BlockInfo &BI = info(N);
+ if (BI.IsHeader) {
+ unsigned I = Headers.size();
+ BI.CycleIdx = I;
+ Headers.push_back(BI.Block);
+ ChildHead.push_back(NoCycle);
+ OwnCount.push_back(1); // The header itself.
+ unsigned &Head = BI.LoopHeader != NoBlock
+ ? ChildHead[info(BI.LoopHeader).CycleIdx]
+ : TopHead;
+ NextSibling.push_back(Head);
+ Head = I;
+ LLVM_DEBUG(errs() << "Found cycle for header: "
+ << Info.Context.print(BI.Block) << "\n");
+ } else if (BI.LoopHeader != NoBlock) {
+ BI.CycleIdx = info(BI.LoopHeader).CycleIdx;
+ ++OwnCount[BI.CycleIdx];
}
-
- // Found a cycle with the candidate as its header.
- LLVM_DEBUG(errs() << "Found cycle for header: "
- << Info.Context.print(HeaderCandidate) << "\n");
- CycleT *NewCycle = &AllCycles.emplace_back();
- NewCycle->IdxBegin = AttachedChildren.size(); // Attach-log slice start.
- NewCycle->appendEntry(HeaderCandidate);
- recordInnermostCycle(HeaderCandidate);
- // The header is this cycle's first own block. Until flatten() runs,
- // IdxEnd accumulates this cycle's own-block count (see the IdxBegin/
- // IdxEnd doc comment), so flatten() needs no separate counting pass.
- ++NewCycle->IdxEnd;
-
- // Helper function to process (non-back-edge) predecessors of a discovered
- // block and either add them to the worklist or recognize that the given
- // block is an additional cycle entry.
- auto ProcessPredecessors = [&](BlockT *Block) {
- LLVM_DEBUG(errs() << " block " << Info.Context.print(Block) << ": ");
-
- bool IsEntry = false;
- for (BlockT *Pred : predecessors(Block)) {
- const DFSInfo PredDFSInfo = getDFSInfo(Pred);
- if (CandidateInfo.isAncestorOf(PredDFSInfo)) {
- Worklist.push_back(Pred);
- } else if (!PredDFSInfo) {
- // Ignore an unreachable predecessor. It will will incorrectly cause
- // Block to be treated as a cycle entry.
- LLVM_DEBUG(errs() << " skipped unreachable predecessor.\n");
- } else {
- IsEntry = true;
- }
- }
- if (IsEntry) {
- assert(!is_contained(NewCycle->Entries, Block));
- LLVM_DEBUG(errs() << "append as entry\n");
- NewCycle->appendEntry(Block);
- } else {
- LLVM_DEBUG(errs() << "append as child\n");
- }
- };
-
- do {
- BlockT *Block = Worklist.pop_back_val();
- if (Block == HeaderCandidate)
- continue;
-
- // If the block has already been discovered by some cycle
- // (possibly by ourself), then the outermost cycle containing it
- // should become our child. Walk the temporary forest directly:
- // handles are not meaningful until flatten() builds the flat array, so
- // BlockMap still holds creation-order indices into AllCycles.
- unsigned Created = Info.BlockMap[GraphTraits<BlockT *>::getNumber(Block)];
- CycleT *BlockParent =
- Created == CycleInfoT::NoCycle ? nullptr : &AllCycles[Created];
- while (BlockParent && BlockParent->hasParent())
- BlockParent = &AllCycles[BlockParent->ParentIndex];
- if (BlockParent) {
- LLVM_DEBUG(errs() << " block " << Info.Context.print(Block) << ": ");
-
- if (BlockParent != NewCycle) {
- LLVM_DEBUG(errs()
- << "discovered child cycle "
- << Info.Context.print(BlockParent->Entries[0]) << "\n");
- // Make BlockParent the child of NewCycle.
- moveTopLevelCycleToNewParent(NewCycle, BlockParent);
-
- for (auto *ChildEntry : BlockParent->Entries)
- ProcessPredecessors(ChildEntry);
- } else {
- LLVM_DEBUG(errs()
- << "known child cycle "
- << Info.Context.print(BlockParent->Entries[0]) << "\n");
- }
- } else {
- recordInnermostCycle(Block);
- ++NewCycle->IdxEnd; // Block's innermost cycle is NewCycle.
- ProcessPredecessors(Block);
- }
- } while (!Worklist.empty());
-
- NewCycle->Depth = AttachedChildren.size(); // Attach-log slice end.
- TopLevelCycles.push_back(NewCycle);
}
+ flatten(Headers, ChildHead, NextSibling, OwnCount, TopHead);
+ if (Reentries.empty())
+ return;
- // The cycle forest and the block-to-innermost-cycle map are complete; move
- // the forest into Info's flat preorder array and lay out every cycle's
- // blocks into the shared contiguous BlockLayout.
- flatten(BlockPreorder);
+ // Add the non-header entries recorded during the DFS. Sorting by preorder
+ // rank appends each cycle's entries in block preorder; several edges may
+ // re-enter a cycle at the same block, so drop duplicates.
+ SmallVector<unsigned, 8> Rank(BlockInfos.size());
+ for (auto [R, N] : enumerate(Preorder))
+ Rank[N] = R;
+ for (auto &[B, H] : Reentries)
+ B = Rank[B];
+ llvm::sort(Reentries);
+ Reentries.erase(llvm::unique(Reentries), Reentries.end());
+ for (auto [R, H] : Reentries)
+ Info.deref(Info.BlockMap[H]).appendEntry(info(Preorder[R]).Block);
}
-/// \brief Compute a DFS of basic blocks starting at the function entry.
-///
-/// Fills BlockDFSInfo with start/end counters and BlockPreorder.
+/// Identify (possibly irreducible) loops using a single-pass DFS algorithm of
+/// "A New Algorithm for Identifying Loops in Decompilation" (SAS 2007). The
+/// cycle forest is then reconstructed from the per-block header tags.
template <typename ContextT>
-void GenericCycleInfoCompute<ContextT>::dfs(FunctionT *F, BlockT *EntryBlock) {
- BlockDFSInfo.resize(GraphTraits<FunctionT *>::getMaxNumber(F));
-
+void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
// Successors are visited in reverse order to match the legacy
// single-LIFO-stack traversal, keeping cycle identification and block order
// unchanged.
using SuccIt = decltype(successors(EntryBlock).begin());
struct Frame {
- BlockT *Block;
+ unsigned Block;
std::reverse_iterator<SuccIt> Cur, End;
};
SmallVector<Frame, 8> Stack;
unsigned Counter = 0;
+ Preorder.reserve(BlockInfos.size());
auto open = [&](BlockT *Block) {
- getOrInsertDFSInfo(Block).Start = ++Counter;
- BlockPreorder.push_back(Block);
- LLVM_DEBUG(errs() << "DFS visiting block: " << Info.Context.print(Block)
- << ", preorder number: " << Counter << "\n");
+ unsigned N = num(Block);
+ BlockInfo &BI = info(N);
+ BI.Block = Block;
+ BI.DFSPPos = ++Counter;
+ Preorder.push_back(N);
auto Succs = successors(Block);
- Stack.push_back({Block, std::make_reverse_iterator(Succs.end()),
+ Stack.push_back({N, std::make_reverse_iterator(Succs.end()),
std::make_reverse_iterator(Succs.begin())});
};
open(EntryBlock);
while (!Stack.empty()) {
Frame &Top = Stack.back();
- BlockT *Next = nullptr;
- while (Top.Cur != Top.End) {
- BlockT *Succ = *Top.Cur++;
- if (getOrInsertDFSInfo(Succ).Start == 0) {
- Next = Succ;
- break;
+ if (Top.Cur != Top.End) {
+ unsigned B0 = Top.Block;
+ BlockT *B1P = *Top.Cur++;
+ unsigned B1 = num(B1P);
+ BlockInfo &B1Info = info(B1);
+ if (!B1Info.Block) {
+ // Tree edge; the weaving happens when B1's frame is popped.
+ open(B1P);
+ } else if (B1Info.DFSPPos > 0) {
+ // B1 is a loop header (including self-edge).
+ B1Info.IsHeader = true;
+ tagLoopHeader(B0, B1);
+ } else {
+ // Climb B1's header chain outward: every enclosing header still off
+ // the DFS path heads a closed cycle that this edge re-enters, making
+ // B1 a non-header entry of it (and the cycle irreducible). Stop at the
+ // first header still on the path (genuine interior node of the loop),
+ // and attribute B0 to it.
+ for (unsigned H = B1Info.LoopHeader; H != NoBlock;
+ H = info(H).LoopHeader) {
+ if (info(H).DFSPPos > 0) {
+ tagLoopHeader(B0, H);
+ break;
+ }
+ Reentries.push_back({B1, H});
+ }
}
- LLVM_DEBUG(errs() << " already visited successor: "
- << Info.Context.print(Succ) << "\n");
- }
- if (Next) {
- open(Next);
} else {
- // Top's subtree is complete. Its end counter is the largest preorder
- // number in the subtree.
- getOrInsertDFSInfo(Top.Block).End = Counter;
- LLVM_DEBUG(errs() << "DFS block " << Info.Context.print(Top.Block)
- << " ended at " << Counter << "\n");
+ // Leave the DFS path.
+ unsigned B0 = Top.Block;
+ info(B0).DFSPPos = 0;
Stack.pop_back();
+ // And weave into the parent's chain (continue the "Tree edge" case).
+ if (!Stack.empty())
+ tagLoopHeader(Stack.back().Block, info(B0).LoopHeader);
}
}
-
- LLVM_DEBUG({
- errs() << "Preorder:\n";
- for (int I = 0, E = BlockPreorder.size(); I != E; ++I)
- errs() << " " << Info.Context.print(BlockPreorder[I]) << ": " << I
- << "\n";
- });
}
/// \brief Reset the object to its initial state.
@@ -597,7 +538,7 @@ void GenericCycleInfo<ContextT>::compute(FunctionT &F) {
GenericCycleInfoCompute<ContextT> Compute(*this);
Context = ContextT(&F);
BlockNumberEpoch = GraphTraits<FunctionT *>::getNumberEpoch(&F);
- BlockMap.assign(GraphTraits<FunctionT *>::getMaxNumber(&F), NoCycle);
+ BlockMap.assign(GraphTraits<FunctionT *>::getMaxNumber(&F), CycleRef());
LLVM_DEBUG(errs() << "Computing cycles for function: " << F.getName()
<< "\n");
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 67c301dd1121c..1245ef5d49bb1 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -55,6 +55,7 @@ class CycleRef {
explicit CycleRef(unsigned Index) : Index(Index) {}
template <typename ContextT> friend class GenericCycleInfo;
+ template <typename ContextT> friend class GenericCycleInfoCompute;
friend struct DenseMapInfo<CycleRef>;
public:
@@ -80,17 +81,12 @@ template <typename ContextT> class GenericCycleInfo {
template <typename> friend class GenericCycleInfoCompute;
private:
- /// Sentinel for a cycle-index slot that refers to no cycle.
- static constexpr unsigned NoCycle = ~0u;
-
/// Internal, data-only storage for a cycle. Consumers name a cycle by a
/// CycleRef handle and query it through GenericCycleInfo.
class Cycle {
public:
- /// Preorder index of the parent cycle, or NoCycle for a top-level
- /// cycle. Before flatten() this holds a creation-order index into
- /// GenericCycleInfoCompute::AllCycles.
- unsigned ParentIndex = NoCycle;
+ /// The parent cycle; invalid for a top-level cycle.
+ CycleRef Parent;
/// The entry block(s) of the cycle. The header is the only entry if this
/// is a loop.
@@ -100,10 +96,6 @@ template <typename ContextT> class GenericCycleInfo {
/// half-open range [IdxBegin, IdxEnd) of BlockLayout, nested like an Euler
/// tour of the cycle tree, so containment is an interval test (see
/// contains()).
- ///
- /// During construction (before the forest is flattened), IdxEnd accumulates
- /// the number of this cycle's own blocks (those whose innermost cycle is
- /// this one).
unsigned IdxBegin = 0, IdxEnd = 0;
/// Depth of the cycle in the tree: top-level cycles are at depth 1 and each
@@ -118,7 +110,7 @@ template <typename ContextT> class GenericCycleInfo {
void appendEntry(BlockT *Block) { Entries.push_back(Block); }
/// Whether this cycle has a parent, i.e. is not top-level.
- bool hasParent() const { return ParentIndex != NoCycle; }
+ bool hasParent() const { return Parent.isValid(); }
Cycle() = default;
Cycle(const Cycle &) = delete;
@@ -131,12 +123,9 @@ template <typename ContextT> class GenericCycleInfo {
ContextT Context;
unsigned BlockNumberEpoch;
- /// Map each basic block number to the preorder index of its inner-most
- /// containing cycle, or NoCycle if none. During construction this
- /// transiently holds creation-order indices into
- /// GenericCycleInfoCompute::AllCycles, which flatten() remaps to preorder
- /// indices.
- SmallVector<unsigned> BlockMap;
+ /// Map each basic block number to its inner-most containing cycle, or an
+ /// invalid handle if none.
+ SmallVector<CycleRef> BlockMap;
/// Euler tour of the cycle forest: every cycle's blocks form a contiguous
/// slice [IdxBegin, IdxEnd) of this array, nested inside its parent's.
@@ -173,7 +162,7 @@ template <typename ContextT> class GenericCycleInfo {
GraphTraits<const FunctionT *>::getNumberEpoch(Fn) &&
"CycleInfo used with outdated block number epoch");
}
- void addToBlockMap(BlockT *Block, CycleT *Cycle);
+ void addToBlockMap(BlockT *Block, CycleRef C);
public:
/// Iteration over child cycles, yielding handles. The first child (if any)
@@ -225,17 +214,14 @@ template <typename ContextT> class GenericCycleInfo {
unsigned Number = GraphTraits<const BlockT *>::getNumber(Block);
// A block added after compute() that no cycle contains (e.g. a critical
// edge MachineSink split outside every cycle) has a number beyond BlockMap.
- if (Number >= BlockMap.size() || BlockMap[Number] == NoCycle)
+ if (Number >= BlockMap.size())
return CycleRef();
- return CycleRef(BlockMap[Number]);
+ return BlockMap[Number];
}
BlockT *getHeader(CycleRef C) const { return deref(C).Entries[0]; }
bool isReducible(CycleRef C) const { return deref(C).Entries.size() == 1; }
- CycleRef getParentCycle(CycleRef C) const {
- auto P = deref(C).ParentIndex;
- return P == NoCycle ? CycleRef() : CycleRef(P);
- }
+ CycleRef getParentCycle(CycleRef C) const { return deref(C).Parent; }
unsigned getDepth(CycleRef C) const { return deref(C).Depth; }
size_t getNumBlocks(CycleRef C) const {
const CycleT &Cyc = deref(C);
diff --git a/llvm/test/Analysis/CycleInfo/basic.ll b/llvm/test/Analysis/CycleInfo/basic.ll
index 240722b9f3788..b96f99aa05f56 100644
--- a/llvm/test/Analysis/CycleInfo/basic.ll
+++ b/llvm/test/Analysis/CycleInfo/basic.ll
@@ -204,7 +204,7 @@ exit:
define void @irreducible_into_simple_cycle(i1 %arg) {
; CHECK-LABEL: CycleInfo for function: irreducible_into_simple_cycle
-; CHECK: depth=1: entries(F C A) B D E
+; CHECK: depth=1: entries(F A C) B D E
entry:
switch i32 undef, label %A [ i32 0, label %C
i32 1, label %F ]
diff --git a/llvm/test/CodeGen/X86/cycle-info.mir b/llvm/test/CodeGen/X86/cycle-info.mir
index 22cfa96f1fac0..a4df89df323ee 100644
--- a/llvm/test/CodeGen/X86/cycle-info.mir
+++ b/llvm/test/CodeGen/X86/cycle-info.mir
@@ -225,9 +225,9 @@ body: |
---
# LEGACY-LABEL: MachineCycleInfo for function: nested_sibling_loops
# NPM-LABEL: name: nested_sibling_loops
-# CHECK: depth=1: entries(bb.1) bb.3 bb.4 bb.5 bb.2
-# CHECK: depth=2: entries(bb.4) bb.5
+# CHECK: depth=1: entries(bb.1) bb.3 bb.2 bb.4 bb.5
# CHECK: depth=2: entries(bb.2)
+# CHECK: depth=2: entries(bb.4) bb.5
name: nested_sibling_loops
alignment: 16
tracksRegLiveness: true
@@ -451,7 +451,7 @@ body: |
---
# LEGACY-LABEL: MachineCycleInfo for function: irreducible_into_simple_cycle
# NPM-LABEL: name: irreducible_into_simple_cycle
-# CHECK: depth=1: entries(bb.2 bb.7 bb.4) bb.3 bb.5 bb.6
+# CHECK: depth=1: entries(bb.2 bb.4 bb.7) bb.3 bb.5 bb.6
name: irreducible_into_simple_cycle
alignment: 16
tracksRegLiveness: true
>From 0814c071c6df5504c04536db1299069da63025b3 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 18 Jul 2026 10:59:59 -0700
Subject: [PATCH 2/6] address comments
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 128 +++++++++++++----------
1 file changed, 72 insertions(+), 56 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index dc45ee19bb8a4..77b8f26c236ae 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -222,29 +222,42 @@ template <typename ContextT> class GenericCycleInfoCompute {
CycleInfoT &Info;
- /// Sentinel header-preorder rank meaning "no cycle".
+ // Sentinel header-preorder rank meaning "no cycle".
static constexpr unsigned NoCycle = ~0u;
- /// Sentinel block number meaning "no block".
+ // Sentinel block number meaning "no block".
static constexpr unsigned NoBlock = ~0u;
- /// Per-block state indexed by block number.
+ // Per-block state indexed by block number. All fields default to zero.
struct BlockInfo {
- /// The block this entry describes; null for an unreachable block.
+ // The block this entry describes; non-null once visited by DFS.
BlockT *Block = nullptr;
- /// Block number of the innermost loop header found so far; NoBlock if none.
- unsigned LoopHeader = NoBlock;
- /// 1-based position on the current DFS path; 0 once the block leaves it.
+ // Block number of the innermost loop header; NoBlock if none. Set to
+ // NoBlock by open() on first visit, then woven by tagLoopHeader.
+ unsigned LoopHeader = 0;
+ // 1-based position on the current DFS path; 0 if off path.
unsigned DFSPPos = 0;
- /// Header-preorder rank of the innermost cycle containing this block (the
- /// cycle it heads if IsHeader); NoCycle if none, or unreachable.
- unsigned CycleIdx = NoCycle;
+ // Header-preorder rank of the innermost cycle containing this block (the
+ // cycle it heads if IsHeader); NoCycle if none. Set by the numbering pass.
+ unsigned CycleIdx = 0;
bool IsHeader = false;
};
+
+ // Per-cycle scratch built in run() and consumed by flatten(), keyed by
+ // header-preorder rank.
+ struct CycleBuild {
+ BlockT *Header;
+ unsigned ChildHead;
+ unsigned NextSibling;
+ unsigned OwnCount;
+ };
+
SmallVector<BlockInfo, 8> BlockInfos;
- /// Reachable block numbers in DFS preorder.
+ // Reachable block numbers in DFS preorder.
SmallVector<unsigned, 8> Preorder;
- /// Records (block B, header H): an edge from outside re-enters the closed
- /// cycle headed by H at B, making B a non-header entry of it.
+ // Number of loop headers found by dfs(), i.e. the number of cycles.
+ unsigned NumHeaders = 0;
+ // Records (header H, block B): an edge from outside re-enters the closed
+ // cycle headed by H at B, making B a non-header entry of it.
SmallVector<std::pair<unsigned, unsigned>, 8> Reentries;
GenericCycleInfoCompute(const GenericCycleInfoCompute &) = delete;
@@ -256,10 +269,10 @@ template <typename ContextT> class GenericCycleInfoCompute {
BlockInfo &info(unsigned Number) { return BlockInfos[Number]; }
- /// Weave loop header \p H (and its own header chain) into the loop header
- /// chain of \p B, keeping the chain ordered from innermost to outermost by
- /// DFS-path position. Building this chain on the fly is why the algorithm
- /// needs no union-find (used in the Havlak algorithm) at all.
+ // Weave loop header \p H (and its own header chain) into the loop header
+ // chain of \p B, keeping the chain ordered from innermost to outermost by
+ // DFS-path position. Building this chain on the fly is why the algorithm
+ // needs no union-find (used in the Havlak algorithm) at all.
void tagLoopHeader(unsigned B, unsigned H) {
if (H == NoBlock)
return;
@@ -283,9 +296,7 @@ template <typename ContextT> class GenericCycleInfoCompute {
}
void dfs(BlockT *EntryBlock);
- void flatten(ArrayRef<BlockT *> Headers, ArrayRef<unsigned> ChildHead,
- ArrayRef<unsigned> NextSibling, ArrayRef<unsigned> OwnCount,
- unsigned TopHead);
+ void flatten(ArrayRef<CycleBuild> Build, unsigned TopHead);
public:
GenericCycleInfoCompute(CycleInfoT &Info) : Info(Info) {}
@@ -336,17 +347,12 @@ void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleRef C) {
/// Lay the discovered cycle forest out into Info's flat preorder array: number
/// the cycles in Euler-tour order, set each one's parent, depth and descendant
/// count, place every block into its innermost cycle's region of BlockLayout,
-/// and fill BlockMap. Arrays are keyed by header-preorder rank.
+/// and fill BlockMap. \p Build is keyed by header-preorder rank.
template <typename ContextT>
-void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<BlockT *> Headers,
- ArrayRef<unsigned> ChildHead,
- ArrayRef<unsigned> NextSibling,
- ArrayRef<unsigned> OwnCount,
+void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<CycleBuild> Build,
unsigned TopHead) {
- unsigned N = Headers.size();
+ unsigned N = Build.size();
Info.NumCycles = N;
- if (!N)
- return;
Info.Cycles = std::make_unique<CycleT[]>(N);
// Walk the cycle forest as an Euler tour. On entry a cycle reserves [Cursor,
@@ -366,18 +372,18 @@ void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<BlockT *> Headers,
CycleT &Flat = Info.Cycles[ID];
Flat.Parent = Parent;
Flat.Depth = Parent ? Info.deref(Parent).Depth + 1 : 1;
- Flat.appendEntry(Headers[C]);
- Cursor += OwnCount[C];
+ Flat.appendEntry(Build[C].Header);
+ Cursor += Build[C].OwnCount;
Flat.IdxBegin = Cursor;
- Stack.push_back({ID, ChildHead[C]});
+ Stack.push_back({ID, Build[C].ChildHead});
};
- for (auto TLC = TopHead; TLC != NoCycle; TLC = NextSibling[TLC]) {
+ for (auto TLC = TopHead; TLC != NoCycle; TLC = Build[TLC].NextSibling) {
enter(TLC, CycleRef());
while (!Stack.empty()) {
Frame &F = Stack.back();
if (F.Child != NoCycle) {
unsigned C = F.Child;
- F.Child = NextSibling[C];
+ F.Child = Build[C].NextSibling;
enter(C, CycleRef(F.Flat));
} else {
CycleT &Flat = Info.Cycles[F.Flat];
@@ -407,50 +413,55 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
BlockInfos.assign(GraphTraits<FunctionT *>::getMaxNumber(F), BlockInfo{});
dfs(EntryBlock);
+ if (!NumHeaders)
+ return;
// Number the cycles by their header's preorder rank and resolve every
// block's innermost cycle in one pass: a block's LoopHeader is a DFS
// ancestor and so already numbered, and parents get smaller ranks than
- // their children.
- SmallVector<BlockT *, 8> Headers;
- SmallVector<unsigned, 8> ChildHead, NextSibling, OwnCount;
+ // their children. dfs() counted the headers, so Build needs one allocation;
+ // the exact reserve also keeps push_back from invalidating Head.
+ SmallVector<CycleBuild, 8> Build;
+ Build.reserve(NumHeaders);
unsigned TopHead = NoCycle;
for (unsigned N : Preorder) {
BlockInfo &BI = info(N);
if (BI.IsHeader) {
- unsigned I = Headers.size();
+ unsigned I = Build.size();
BI.CycleIdx = I;
- Headers.push_back(BI.Block);
- ChildHead.push_back(NoCycle);
- OwnCount.push_back(1); // The header itself.
unsigned &Head = BI.LoopHeader != NoBlock
- ? ChildHead[info(BI.LoopHeader).CycleIdx]
+ ? Build[info(BI.LoopHeader).CycleIdx].ChildHead
: TopHead;
- NextSibling.push_back(Head);
+ Build.push_back({BI.Block, NoCycle, Head, 1}); // OwnCount 1: the header.
Head = I;
- LLVM_DEBUG(errs() << "Found cycle for header: "
+ LLVM_DEBUG(dbgs() << "Found cycle for header: "
<< Info.Context.print(BI.Block) << "\n");
} else if (BI.LoopHeader != NoBlock) {
BI.CycleIdx = info(BI.LoopHeader).CycleIdx;
- ++OwnCount[BI.CycleIdx];
+ ++Build[BI.CycleIdx].OwnCount;
+ } else {
+ BI.CycleIdx = NoCycle;
}
}
- flatten(Headers, ChildHead, NextSibling, OwnCount, TopHead);
+ flatten(Build, TopHead);
if (Reentries.empty())
return;
- // Add the non-header entries recorded during the DFS. Sorting by preorder
- // rank appends each cycle's entries in block preorder; several edges may
- // re-enter a cycle at the same block, so drop duplicates.
+ // Add the non-header entries recorded during the DFS. Sorting by (header,
+ // block) groups each cycle's entries together and in block preorder; a block
+ // may re-enter a cycle via several edges, so skip duplicates.
SmallVector<unsigned, 8> Rank(BlockInfos.size());
for (auto [R, N] : enumerate(Preorder))
Rank[N] = R;
- for (auto &[B, H] : Reentries)
+ for (auto &[H, B] : Reentries)
B = Rank[B];
llvm::sort(Reentries);
- Reentries.erase(llvm::unique(Reentries), Reentries.end());
- for (auto [R, H] : Reentries)
+ for (unsigned I = 0, E = Reentries.size(); I != E; ++I) {
+ if (I && Reentries[I] == Reentries[I - 1])
+ continue;
+ auto [H, R] = Reentries[I];
Info.deref(Info.BlockMap[H]).appendEntry(info(Preorder[R]).Block);
+ }
}
/// Identify (possibly irreducible) loops using a single-pass DFS algorithm of
@@ -468,14 +479,15 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
};
SmallVector<Frame, 8> Stack;
unsigned Counter = 0;
- Preorder.reserve(BlockInfos.size());
+ Preorder.resize_for_overwrite(BlockInfos.size());
auto open = [&](BlockT *Block) {
unsigned N = num(Block);
BlockInfo &BI = info(N);
BI.Block = Block;
+ BI.LoopHeader = NoBlock;
BI.DFSPPos = ++Counter;
- Preorder.push_back(N);
+ Preorder[Counter - 1] = N;
auto Succs = successors(Block);
Stack.push_back({N, std::make_reverse_iterator(Succs.end()),
std::make_reverse_iterator(Succs.begin())});
@@ -494,7 +506,10 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
open(B1P);
} else if (B1Info.DFSPPos > 0) {
// B1 is a loop header (including self-edge).
- B1Info.IsHeader = true;
+ if (!B1Info.IsHeader) {
+ B1Info.IsHeader = true;
+ ++NumHeaders;
+ }
tagLoopHeader(B0, B1);
} else {
// Climb B1's header chain outward: every enclosing header still off
@@ -508,7 +523,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
tagLoopHeader(B0, H);
break;
}
- Reentries.push_back({B1, H});
+ Reentries.push_back({H, B1});
}
}
} else {
@@ -521,6 +536,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
tagLoopHeader(Stack.back().Block, info(B0).LoopHeader);
}
}
+ Preorder.truncate(Counter);
}
/// \brief Reset the object to its initial state.
@@ -540,7 +556,7 @@ void GenericCycleInfo<ContextT>::compute(FunctionT &F) {
BlockNumberEpoch = GraphTraits<FunctionT *>::getNumberEpoch(&F);
BlockMap.assign(GraphTraits<FunctionT *>::getMaxNumber(&F), CycleRef());
- LLVM_DEBUG(errs() << "Computing cycles for function: " << F.getName()
+ LLVM_DEBUG(dbgs() << "Computing cycles for function: " << F.getName()
<< "\n");
Compute.run(&F);
}
>From 71a1a6c3c820e9cdf366793f9bd559fc43ee7c31 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 18 Jul 2026 11:49:48 -0700
Subject: [PATCH 3/6] layout
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 77b8f26c236ae..af79ea4034a9b 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -231,11 +231,11 @@ template <typename ContextT> class GenericCycleInfoCompute {
struct BlockInfo {
// The block this entry describes; non-null once visited by DFS.
BlockT *Block = nullptr;
+ // 1-based position on the current DFS path; 0 if off path.
+ unsigned DFSPPos = 0;
// Block number of the innermost loop header; NoBlock if none. Set to
// NoBlock by open() on first visit, then woven by tagLoopHeader.
unsigned LoopHeader = 0;
- // 1-based position on the current DFS path; 0 if off path.
- unsigned DFSPPos = 0;
// Header-preorder rank of the innermost cycle containing this block (the
// cycle it heads if IsHeader); NoCycle if none. Set by the numbering pass.
unsigned CycleIdx = 0;
@@ -254,7 +254,7 @@ template <typename ContextT> class GenericCycleInfoCompute {
SmallVector<BlockInfo, 8> BlockInfos;
// Reachable block numbers in DFS preorder.
SmallVector<unsigned, 8> Preorder;
- // Number of loop headers found by dfs(), i.e. the number of cycles.
+ // Number of loop headers found by dfs().
unsigned NumHeaders = 0;
// Records (header H, block B): an edge from outside re-enters the closed
// cycle headed by H at B, making B a non-header entry of it.
@@ -419,9 +419,9 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
// Number the cycles by their header's preorder rank and resolve every
// block's innermost cycle in one pass: a block's LoopHeader is a DFS
// ancestor and so already numbered, and parents get smaller ranks than
- // their children. dfs() counted the headers, so Build needs one allocation;
- // the exact reserve also keeps push_back from invalidating Head.
+ // their children.
SmallVector<CycleBuild, 8> Build;
+ // Exact reserve so the Head reference below survives each push_back.
Build.reserve(NumHeaders);
unsigned TopHead = NoCycle;
for (unsigned N : Preorder) {
@@ -483,11 +483,11 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
auto open = [&](BlockT *Block) {
unsigned N = num(Block);
+ Preorder[Counter] = N;
BlockInfo &BI = info(N);
BI.Block = Block;
- BI.LoopHeader = NoBlock;
BI.DFSPPos = ++Counter;
- Preorder[Counter - 1] = N;
+ BI.LoopHeader = NoBlock;
auto Succs = successors(Block);
Stack.push_back({N, std::make_reverse_iterator(Succs.end()),
std::make_reverse_iterator(Succs.begin())});
@@ -512,11 +512,10 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
}
tagLoopHeader(B0, B1);
} else {
- // Climb B1's header chain outward: every enclosing header still off
- // the DFS path heads a closed cycle that this edge re-enters, making
- // B1 a non-header entry of it (and the cycle irreducible). Stop at the
- // first header still on the path (genuine interior node of the loop),
- // and attribute B0 to it.
+ // Climb B1's header chain: each enclosing header still off the DFS path
+ // heads a closed cycle this edge re-enters, so B1 is a non-header entry
+ // of it (and it is irreducible). Stop at the first on-path header and
+ // attribute B0 to it.
for (unsigned H = B1Info.LoopHeader; H != NoBlock;
H = info(H).LoopHeader) {
if (info(H).DFSPPos > 0) {
>From 6ec5019008d94e5dd54f1aae0cbc597c96682e75 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 18 Jul 2026 12:06:56 -0700
Subject: [PATCH 4/6] optimize tagLoopHeader
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index af79ea4034a9b..7609d40b0dcca 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -274,8 +274,7 @@ template <typename ContextT> class GenericCycleInfoCompute {
// DFS-path position. Building this chain on the fly is why the algorithm
// needs no union-find (used in the Havlak algorithm) at all.
void tagLoopHeader(unsigned B, unsigned H) {
- if (H == NoBlock)
- return;
+ assert(H != NoBlock);
// Invariant: info(B).DFSPPos >= info(H).DFSPPos.
while (B != H) {
unsigned IH = info(B).LoopHeader;
@@ -531,7 +530,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
info(B0).DFSPPos = 0;
Stack.pop_back();
// And weave into the parent's chain (continue the "Tree edge" case).
- if (!Stack.empty())
+ if (!Stack.empty() && info(B0).LoopHeader != NoBlock)
tagLoopHeader(Stack.back().Block, info(B0).LoopHeader);
}
}
>From 88daa24713d55e71251d051d25200f259ec8bafd Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 18 Jul 2026 13:52:46 -0700
Subject: [PATCH 5/6] Shrink BlockInfo; BlockInfos.assign now optimizes to zero
memset
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 70 ++++++++++++++----------
1 file changed, 40 insertions(+), 30 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 7609d40b0dcca..589c868529dc6 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -26,6 +26,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/GenericCycleInfo.h"
+#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include <iterator>
@@ -229,17 +230,26 @@ template <typename ContextT> class GenericCycleInfoCompute {
// Per-block state indexed by block number. All fields default to zero.
struct BlockInfo {
- // The block this entry describes; non-null once visited by DFS.
- BlockT *Block = nullptr;
- // 1-based position on the current DFS path; 0 if off path.
- unsigned DFSPPos = 0;
+ // The block this entry describes (non-null once visited by DFS), packed
+ // with a bit for whether it heads a loop.
+ PointerIntPair<BlockT *, 1, bool> BlockAndHeader;
+ union {
+ // (Live only during DFS) 1-based position on the current DFS path; 0 if
+ // off path.
+ unsigned Pos = 0;
+ // (Live after DFS) Header-preorder rank of the innermost
+ // loop containing this block (the loop it heads if IsHeader); NoCycle if
+ // none.
+ unsigned LoopIdx;
+ };
// Block number of the innermost loop header; NoBlock if none. Set to
// NoBlock by open() on first visit, then woven by tagLoopHeader.
unsigned LoopHeader = 0;
- // Header-preorder rank of the innermost cycle containing this block (the
- // cycle it heads if IsHeader); NoCycle if none. Set by the numbering pass.
- unsigned CycleIdx = 0;
- bool IsHeader = false;
+
+ BlockT *getBlock() const { return BlockAndHeader.getPointer(); }
+ void setBlock(BlockT *B) { BlockAndHeader.setPointer(B); }
+ bool isHeader() const { return BlockAndHeader.getInt(); }
+ void setHeader() { BlockAndHeader.setInt(true); }
};
// Per-cycle scratch built in run() and consumed by flatten(), keyed by
@@ -275,7 +285,7 @@ template <typename ContextT> class GenericCycleInfoCompute {
// needs no union-find (used in the Havlak algorithm) at all.
void tagLoopHeader(unsigned B, unsigned H) {
assert(H != NoBlock);
- // Invariant: info(B).DFSPPos >= info(H).DFSPPos.
+ // Invariant: info(B).Pos >= info(H).Pos.
while (B != H) {
unsigned IH = info(B).LoopHeader;
if (IH == NoBlock) {
@@ -284,7 +294,7 @@ template <typename ContextT> class GenericCycleInfoCompute {
return;
}
// Keep whichever candidate header is inner (larger DFS-path position).
- if (info(IH).DFSPPos >= info(H).DFSPPos)
+ if (info(IH).Pos >= info(H).Pos)
B = IH;
else {
info(B).LoopHeader = H;
@@ -397,11 +407,11 @@ void GenericCycleInfoCompute<ContextT>::flatten(ArrayRef<CycleBuild> Build,
Info.BlockLayout.resize_for_overwrite(Cursor);
for (unsigned N : llvm::reverse(Preorder)) {
BlockInfo &BI = info(N);
- if (BI.CycleIdx == NoCycle)
+ if (BI.LoopIdx == NoCycle)
continue;
- unsigned Flat = FlatIdx[BI.CycleIdx];
+ unsigned Flat = FlatIdx[BI.LoopIdx];
Info.BlockMap[N] = CycleRef(Flat);
- Info.BlockLayout[--Info.Cycles[Flat].IdxBegin] = BI.Block;
+ Info.BlockLayout[--Info.Cycles[Flat].IdxBegin] = BI.getBlock();
}
}
@@ -425,21 +435,21 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
unsigned TopHead = NoCycle;
for (unsigned N : Preorder) {
BlockInfo &BI = info(N);
- if (BI.IsHeader) {
+ if (BI.isHeader()) {
unsigned I = Build.size();
- BI.CycleIdx = I;
+ BI.LoopIdx = I;
unsigned &Head = BI.LoopHeader != NoBlock
- ? Build[info(BI.LoopHeader).CycleIdx].ChildHead
+ ? Build[info(BI.LoopHeader).LoopIdx].ChildHead
: TopHead;
- Build.push_back({BI.Block, NoCycle, Head, 1}); // OwnCount 1: the header.
+ Build.push_back({BI.getBlock(), NoCycle, Head, 1}); // OwnCount 1: header.
Head = I;
LLVM_DEBUG(dbgs() << "Found cycle for header: "
- << Info.Context.print(BI.Block) << "\n");
+ << Info.Context.print(BI.getBlock()) << "\n");
} else if (BI.LoopHeader != NoBlock) {
- BI.CycleIdx = info(BI.LoopHeader).CycleIdx;
- ++Build[BI.CycleIdx].OwnCount;
+ BI.LoopIdx = info(BI.LoopHeader).LoopIdx;
+ ++Build[BI.LoopIdx].OwnCount;
} else {
- BI.CycleIdx = NoCycle;
+ BI.LoopIdx = NoCycle;
}
}
flatten(Build, TopHead);
@@ -459,7 +469,7 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
if (I && Reentries[I] == Reentries[I - 1])
continue;
auto [H, R] = Reentries[I];
- Info.deref(Info.BlockMap[H]).appendEntry(info(Preorder[R]).Block);
+ Info.deref(Info.BlockMap[H]).appendEntry(info(Preorder[R]).getBlock());
}
}
@@ -484,8 +494,8 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
unsigned N = num(Block);
Preorder[Counter] = N;
BlockInfo &BI = info(N);
- BI.Block = Block;
- BI.DFSPPos = ++Counter;
+ BI.setBlock(Block);
+ BI.Pos = ++Counter;
BI.LoopHeader = NoBlock;
auto Succs = successors(Block);
Stack.push_back({N, std::make_reverse_iterator(Succs.end()),
@@ -500,13 +510,13 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
BlockT *B1P = *Top.Cur++;
unsigned B1 = num(B1P);
BlockInfo &B1Info = info(B1);
- if (!B1Info.Block) {
+ if (!B1Info.getBlock()) {
// Tree edge; the weaving happens when B1's frame is popped.
open(B1P);
- } else if (B1Info.DFSPPos > 0) {
+ } else if (B1Info.Pos > 0) {
// B1 is a loop header (including self-edge).
- if (!B1Info.IsHeader) {
- B1Info.IsHeader = true;
+ if (!B1Info.isHeader()) {
+ B1Info.setHeader();
++NumHeaders;
}
tagLoopHeader(B0, B1);
@@ -517,7 +527,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
// attribute B0 to it.
for (unsigned H = B1Info.LoopHeader; H != NoBlock;
H = info(H).LoopHeader) {
- if (info(H).DFSPPos > 0) {
+ if (info(H).Pos > 0) {
tagLoopHeader(B0, H);
break;
}
@@ -527,7 +537,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
} else {
// Leave the DFS path.
unsigned B0 = Top.Block;
- info(B0).DFSPPos = 0;
+ info(B0).Pos = 0;
Stack.pop_back();
// And weave into the parent's chain (continue the "Tree edge" case).
if (!Stack.empty() && info(B0).LoopHeader != NoBlock)
>From 03b5987036bbb1176eebab7fbddf7e2c5fbcb0f9 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sat, 18 Jul 2026 20:06:04 -0700
Subject: [PATCH 6/6] Optimize open() and visited() in dfs()
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 3e7273d590f87..bbc6230aefe5f 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -233,9 +233,10 @@ template <typename ContextT> class GenericCycleInfoCompute {
unsigned LoopHeader = 0;
BlockT *getBlock() const { return BlockAndHeader.getPointer(); }
- void setBlock(BlockT *B) { BlockAndHeader.setPointer(B); }
bool isHeader() const { return BlockAndHeader.getInt(); }
void setHeader() { BlockAndHeader.setInt(true); }
+ // An unvisited entry is all-zero,
+ bool visited() const { return BlockAndHeader.getOpaqueValue() != nullptr; }
};
// Per-cycle scratch built in run() and consumed by flatten(), keyed by
@@ -480,7 +481,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
unsigned N = num(Block);
Preorder[Counter] = N;
BlockInfo &BI = info(N);
- BI.setBlock(Block);
+ BI.BlockAndHeader.setPointerAndInt(Block, false);
BI.Pos = ++Counter;
BI.LoopHeader = NoBlock;
auto Succs = successors(Block);
@@ -496,7 +497,7 @@ void GenericCycleInfoCompute<ContextT>::dfs(BlockT *EntryBlock) {
BlockT *B1P = *Top.Cur++;
unsigned B1 = num(B1P);
BlockInfo &B1Info = info(B1);
- if (!B1Info.getBlock()) {
+ if (!B1Info.visited()) {
// Tree edge; the weaving happens when B1's frame is popped.
open(B1P);
} else if (B1Info.Pos > 0) {
More information about the llvm-commits
mailing list