[Mlir-commits] [llvm] [mlir] [LoopInfo] Store blocks using Euler tour representation (PR #211485)
Fangrui Song
llvmlistbot at llvm.org
Thu Jul 23 01:52:23 PDT 2026
https://github.com/MaskRay created https://github.com/llvm/llvm-project/pull/211485
Block list construction appends each block to all enclosing loops
(O(depth) per block), and each loop owns a separately allocated vector.
Switch to an Euler tour representation: one BlockLayout array per
LoopInfo, each loop's list a [begin, end) slice of it, subloop slices
nested inside their parent's.
Mutations first copy a borrowed slice into private storage from
LoopInfo's allocator. `contains(BlockT *)` remains map-based: a
materialized loop's slice is not a membership test.
Previously a loop's list was the function's reverse postorder restricted
to its members, so a subloop's blocks were interleaved with the parent's
own blocks at their RPO positions. Now a subloop's blocks are contiguous
after its header within every ancestor's list. Headers remain first;
`SubLoops`/`TopLevelLoops` orders are unchanged. A few tests observe the
order and are updated.
AllocateLoop() no longer forwards constructor arguments; analyze() uses
allocateLoop(Header), removing the header-taking loop constructors.
Aided by Claude Fable 5
>From fff8f4d18de1b8d5d6d7d4234b7f8f405a7e9cc8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Wed, 22 Jul 2026 21:51:02 -0700
Subject: [PATCH] [LoopInfo] Store blocks using Euler tour representation
Block list construction appends each block to all enclosing loops
(O(depth) per block), and each loop owns a separately allocated vector.
Switch to an Euler tour representation: one BlockLayout array per
LoopInfo, each loop's list a [begin, end) slice of it, subloop slices
nested inside their parent's.
Mutations first copy a borrowed slice into private storage from
LoopInfo's allocator. `contains(BlockT *)` remains map-based: a
materialized loop's slice is not a membership test.
Previously a loop's list was the function's reverse postorder restricted
to its members, so a subloop's blocks were interleaved with the parent's
own blocks at their RPO positions. Now a subloop's blocks are contiguous
after its header within every ancestor's list. Headers remain first;
`SubLoops`/`TopLevelLoops` orders are unchanged. A few tests observe the
order and are updated.
AllocateLoop() no longer forwards constructor arguments; analyze() uses
allocateLoop(Header), removing the header-taking loop constructors.
Aided by Claude Fable 5
---
bolt/include/bolt/Core/BinaryLoop.h | 5 -
bolt/test/X86/loop-nest.test | 2 +-
llvm/include/llvm/Analysis/LoopInfo.h | 1 -
llvm/include/llvm/CodeGen/MachineLoopInfo.h | 3 -
llvm/include/llvm/Support/GenericLoopInfo.h | 126 +++++++++++----
.../llvm/Support/GenericLoopInfoImpl.h | 153 +++++++++---------
.../early_exit_with_outer_loop.ll | 4 +-
.../nontrivial-unswitch-freeze.ll | 26 +--
.../SimpleLoopUnswitch/nontrivial-unswitch.ll | 6 +-
mlir/include/mlir/Analysis/CFGLoopInfo.h | 2 +-
mlir/lib/Analysis/CFGLoopInfo.cpp | 3 -
11 files changed, 186 insertions(+), 145 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryLoop.h b/bolt/include/bolt/Core/BinaryLoop.h
index b425c75715d8b..1c52a051a84cc 100644
--- a/bolt/include/bolt/Core/BinaryLoop.h
+++ b/bolt/include/bolt/Core/BinaryLoop.h
@@ -36,11 +36,6 @@ class BinaryLoop : public LoopBase<BinaryBasicBlock, BinaryLoop> {
uint64_t ExitCount{0};
// Most of the public interface is provided by LoopBase.
-
-protected:
- friend class LoopInfoBase<BinaryBasicBlock, BinaryLoop>;
- explicit BinaryLoop(BinaryBasicBlock *BB)
- : LoopBase<BinaryBasicBlock, BinaryLoop>(BB) {}
};
class BinaryLoopInfo : public LoopInfoBase<BinaryBasicBlock, BinaryLoop> {
diff --git a/bolt/test/X86/loop-nest.test b/bolt/test/X86/loop-nest.test
index 51c8fcdb32eaa..208cc9ebdfaaf 100644
--- a/bolt/test/X86/loop-nest.test
+++ b/bolt/test/X86/loop-nest.test
@@ -18,6 +18,6 @@ CHECK-NEXT: Loop basic blocks: .Ltmp[[#MAIN_OUTER_HDR]], .Ltmp[[#]]
CHECK: Binary Function "foo" after building cfg
CHECK: Loop Info for Function "foo"
CHECK: Outer loop header: .Ltmp[[#FOO_OUTER_HDR:]]
-CHECK-NEXT: Loop basic blocks: .Ltmp[[#FOO_OUTER_HDR]], .Ltmp[[#]], .LFT[[#]], .Ltmp[[#]], .Ltmp[[#FOO_INNER_HDR:]], .Ltmp[[#]], .Ltmp[[#FOO_INNER_BB:]]
+CHECK-NEXT: Loop basic blocks: .Ltmp[[#FOO_OUTER_HDR]], .Ltmp[[#]], .LFT[[#]], .Ltmp[[#]], .Ltmp[[#FOO_INNER_HDR:]], .Ltmp[[#FOO_INNER_BB:]], .Ltmp[[#]]
CHECK: Nested loop header: .Ltmp[[#FOO_INNER_HDR]]
CHECK-NEXT: Loop basic blocks: .Ltmp[[#FOO_INNER_HDR]], .Ltmp[[#FOO_INNER_BB]]
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index 13b404505b5e2..15c22e43c44bf 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -417,7 +417,6 @@ class LLVM_ABI Loop : public LoopBase<BasicBlock, Loop> {
friend class LoopInfoBase<BasicBlock, Loop>;
friend class LoopBase<BasicBlock, Loop>;
- explicit Loop(BasicBlock *BB) : LoopBase<BasicBlock, Loop>(BB) {}
~Loop() = default;
};
diff --git a/llvm/include/llvm/CodeGen/MachineLoopInfo.h b/llvm/include/llvm/CodeGen/MachineLoopInfo.h
index 60b3f6c6a3ba5..7893f70cd353c 100644
--- a/llvm/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineLoopInfo.h
@@ -96,9 +96,6 @@ class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
/// Returns true if the given physreg has no defs inside the loop.
bool isLoopInvariantImplicitPhysReg(Register Reg) const;
- explicit MachineLoop(MachineBasicBlock *MBB)
- : LoopBase<MachineBasicBlock, MachineLoop>(MBB) {}
-
MachineLoop() = default;
};
diff --git a/llvm/include/llvm/Support/GenericLoopInfo.h b/llvm/include/llvm/Support/GenericLoopInfo.h
index 0bdfce7dc889e..cda6577deb31a 100644
--- a/llvm/include/llvm/Support/GenericLoopInfo.h
+++ b/llvm/include/llvm/Support/GenericLoopInfo.h
@@ -51,7 +51,6 @@ namespace llvm {
template <class N, class M> class LoopInfoBase;
template <class N, class M> class LoopBase;
-template <class N, class M> class PopulateLoopsDFS;
//===----------------------------------------------------------------------===//
/// Instances of this class are used to represent loops that are detected in the
@@ -62,8 +61,12 @@ template <class BlockT, class LoopT> class LoopBase {
// Loops contained entirely within this one.
std::vector<LoopT *> SubLoops;
- // The list of blocks in this loop. First entry is the header node.
- std::vector<BlockT *> Blocks;
+ // The list of blocks in this loop; first entry is the header. Either borrows
+ // from the owning LoopInfo's BlockLayout with BlockCapacity == BlockLen, or
+ // is a private allocation of BlockCapacity slots from its allocator.
+ BlockT **BlockData = nullptr;
+ unsigned BlockLen = 0;
+ unsigned BlockCapacity = 0;
// The LoopInfo that owns this loop. Used to answer contains(BlockT *) from
// the central block-to-loop map.
@@ -186,7 +189,7 @@ template <class BlockT, class LoopT> class LoopBase {
/// Get a list of the basic blocks which make up this loop.
ArrayRef<BlockT *> getBlocks() const {
assert(!isInvalid() && "Loop not in a valid state!");
- return Blocks;
+ return ArrayRef<BlockT *>(BlockData, BlockLen);
}
using block_iterator = typename ArrayRef<BlockT *>::const_iterator;
block_iterator block_begin() const { return getBlocks().begin(); }
@@ -200,7 +203,7 @@ template <class BlockT, class LoopT> class LoopBase {
/// Invalidate the loop, indicating that it is no longer a loop.
unsigned getNumBlocks() const {
assert(!isInvalid() && "Loop not in a valid state!");
- return Blocks.size();
+ return BlockLen;
}
/// Return true if this loop is no longer valid. The only valid use of this
@@ -400,19 +403,17 @@ template <class BlockT, class LoopT> class LoopBase {
/// transformations should use addBasicBlockToLoop.
void addBlockEntry(BlockT *BB) {
assert(!isInvalid() && "Loop not in a valid state!");
- Blocks.push_back(BB);
- }
-
- /// interface to reverse Blocks[from, end of loop] in this loop
- void reverseBlock(unsigned from) {
- assert(!isInvalid() && "Loop not in a valid state!");
- std::reverse(Blocks.begin() + from, Blocks.end());
+ if (BlockLen == BlockCapacity)
+ LI->reallocBlocks(*static_cast<LoopT *>(this),
+ std::max(2 * BlockLen, 4u));
+ BlockData[BlockLen++] = BB;
}
/// interface to do reserve() for Blocks
- void reserveBlocks(unsigned size) {
+ void reserveBlocks(unsigned Size) {
assert(!isInvalid() && "Loop not in a valid state!");
- Blocks.reserve(size);
+ if (BlockCapacity < Size)
+ LI->reallocBlocks(*static_cast<LoopT *>(this), Size);
}
/// interface to do reserve() for SubLoops
@@ -421,21 +422,18 @@ template <class BlockT, class LoopT> class LoopBase {
SubLoops.reserve(Size);
}
- /// Capacity of the block list; input to an enclosing loop's reserveBlocks()
- /// during construction, when this loop's list is not yet fully populated.
- unsigned getBlocksCapacity() const { return Blocks.capacity(); }
-
/// This method is used to move BB (which must be part of this loop) to be the
/// loop header of the loop (the block that dominates all others).
void moveToHeader(BlockT *BB) {
assert(!isInvalid() && "Loop not in a valid state!");
- if (Blocks[0] == BB)
+ if (BlockData[0] == BB)
return;
+ LI->materializeBlocks(*static_cast<LoopT *>(this));
for (unsigned i = 0;; ++i) {
- assert(i != Blocks.size() && "Loop does not contain BB!");
- if (Blocks[i] == BB) {
- Blocks[i] = Blocks[0];
- Blocks[0] = BB;
+ assert(i != BlockLen && "Loop does not contain BB!");
+ if (BlockData[i] == BB) {
+ BlockData[i] = BlockData[0];
+ BlockData[0] = BB;
return;
}
}
@@ -446,9 +444,12 @@ template <class BlockT, class LoopT> class LoopBase {
/// class.
void removeBlockFromLoop(BlockT *BB) {
assert(!isInvalid() && "Loop not in a valid state!");
- auto I = find(Blocks, BB);
+ LI->materializeBlocks(*static_cast<LoopT *>(this));
+ MutableArrayRef<BlockT *> Blocks(BlockData, BlockLen);
+ auto *I = llvm::find(Blocks, BB);
assert(I != Blocks.end() && "N is not in this list!");
- Blocks.erase(I);
+ std::move(I + 1, Blocks.end(), I);
+ --BlockLen;
}
/// Verify loop structure
@@ -469,15 +470,10 @@ template <class BlockT, class LoopT> class LoopBase {
protected:
friend class LoopInfoBase<BlockT, LoopT>;
- friend class PopulateLoopsDFS<BlockT, LoopT>;
/// This creates an empty loop.
LoopBase() : ParentLoop(nullptr) {}
- explicit LoopBase(BlockT *BB) : ParentLoop(nullptr) {
- Blocks.push_back(BB);
- }
-
// Since loop passes like SCEV are allowed to key analysis results off of
// `Loop` pointers, we cannot re-use pointers within a loop pass manager.
// This means loop passes should not be `delete` ing `Loop` objects directly
@@ -495,7 +491,10 @@ template <class BlockT, class LoopT> class LoopBase {
IsInvalid = true;
#endif
SubLoops.clear();
- Blocks.clear();
+ // The block storage is reclaimed by the owning LoopInfo.
+ BlockData = nullptr;
+ BlockLen = 0;
+ BlockCapacity = 0;
ParentLoop = nullptr;
}
};
@@ -525,6 +524,12 @@ template <class BlockT, class LoopT> class LoopInfoBase {
unsigned BlockNumberEpoch;
std::vector<LoopT *> TopLevelLoops;
+
+ // Shared reverse postorder layout of all blocks. Each initial loop is a slice
+ // of this array, subloop slices nested inside their parent's.
+ std::unique_ptr<BlockT *[]> BlockLayout;
+ unsigned BlockLayoutLen = 0;
+
BumpPtrAllocator LoopAllocator;
friend class LoopBase<BlockT, LoopT>;
@@ -540,9 +545,11 @@ template <class BlockT, class LoopT> class LoopInfoBase {
LoopInfoBase(LoopInfoBase &&Arg)
: BBMap(std::move(Arg.BBMap)),
TopLevelLoops(std::move(Arg.TopLevelLoops)),
+ BlockLayout(std::move(Arg.BlockLayout)),
LoopAllocator(std::move(Arg.LoopAllocator)) {
ParentPtr = Arg.ParentPtr;
BlockNumberEpoch = Arg.BlockNumberEpoch;
+ BlockLayoutLen = Arg.BlockLayoutLen;
resetLoopInfoOwners();
// We have to clear the arguments top level loops as we've taken ownership.
Arg.TopLevelLoops.clear();
@@ -556,9 +563,12 @@ template <class BlockT, class LoopT> class LoopInfoBase {
L->~LoopT();
TopLevelLoops = std::move(RHS.TopLevelLoops);
+ BlockLayout = std::move(RHS.BlockLayout);
+ BlockLayoutLen = RHS.BlockLayoutLen;
LoopAllocator = std::move(RHS.LoopAllocator);
resetLoopInfoOwners();
RHS.TopLevelLoops.clear();
+ RHS.BlockLayoutLen = 0;
return *this;
}
@@ -568,12 +578,14 @@ template <class BlockT, class LoopT> class LoopInfoBase {
for (auto *L : TopLevelLoops)
L->~LoopT();
TopLevelLoops.clear();
+ BlockLayout.reset();
+ BlockLayoutLen = 0;
LoopAllocator.Reset();
}
- template <typename... ArgsTy> LoopT *AllocateLoop(ArgsTy &&...Args) {
+ LoopT *AllocateLoop() {
LoopT *Storage = LoopAllocator.Allocate<LoopT>();
- LoopT *L = new (Storage) LoopT(std::forward<ArgsTy>(Args)...);
+ LoopT *L = new (Storage) LoopT();
L->LI = this;
return L;
}
@@ -636,6 +648,49 @@ template <class BlockT, class LoopT> class LoopInfoBase {
return Number < BBMap.size() ? BBMap[Number] : nullptr;
}
+ /// AllocateLoop for analyze(): stash \p Header (see pendingHeader).
+ /// getHeader() only works once the layout carve has replaced the stash with
+ /// the loop's block list.
+ LoopT *allocateLoop(BlockT *Header) {
+ LoopT *L = AllocateLoop();
+ L->BlockData = reinterpret_cast<BlockT **>(Header);
+ return L;
+ }
+
+ /// The header of a loop under construction, stashed in BlockData until the
+ /// layout carve builds the block list.
+ static BlockT *pendingHeader(const LoopT *L) {
+ return reinterpret_cast<BlockT *>(L->BlockData);
+ }
+
+ void discoverAndMapSubloop(LoopT *L, BlockT *Header,
+ ArrayRef<BlockT *> Backedges,
+ const DominatorTreeBase<BlockT, false> &DomTree);
+
+ /// True if \p L's block list is a borrowed slice of BlockLayout rather than
+ /// a private allocation.
+ bool hasBorrowedBlocks(const LoopT &L) const {
+ return L.BlockData >= BlockLayout.get() &&
+ L.BlockData < BlockLayout.get() + BlockLayoutLen;
+ }
+
+ /// Replace \p L's block list with a private allocation of NewCapacity
+ /// slots. The old storage is abandoned in place so slices sharing it stay
+ /// intact; it is reclaimed when this LoopInfo is cleared.
+ void reallocBlocks(LoopT &L, unsigned NewCapacity) {
+ assert(NewCapacity >= L.BlockLen && "capacity below size");
+ BlockT **New = LoopAllocator.Allocate<BlockT *>(NewCapacity);
+ llvm::copy(L.getBlocks(), New);
+ L.BlockData = New;
+ L.BlockCapacity = NewCapacity;
+ }
+
+ /// Copy \p L's borrowed block list into private storage before a mutation.
+ void materializeBlocks(LoopT &L) {
+ if (hasBorrowedBlocks(L))
+ reallocBlocks(L, L.BlockLen);
+ }
+
public:
/// Return the inner most loop that BB lives in. If a basic block is in no
/// loop (for example the entry node), null is returned.
@@ -672,7 +727,10 @@ template <class BlockT, class LoopT> class LoopInfoBase {
/// ancestors or descendants, and not the block-to-loop mapping.
template <typename PredicateT>
void removeBlocksIf(LoopT &L, PredicateT Pred) {
- llvm::erase_if(L.Blocks, Pred);
+ materializeBlocks(L);
+ L.BlockLen = llvm::remove_if(
+ MutableArrayRef<BlockT *>(L.BlockData, L.BlockLen), Pred) -
+ L.BlockData;
}
/// Remove every block satisfying \p Pred from \p Start and each of its
diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
index 170f05e9d3a49..ad36d3343c8a8 100644
--- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h
+++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
@@ -282,7 +282,7 @@ void LoopBase<BlockT, LoopT>::addBasicBlockToLoop(
BlockT *NewBB, LoopInfoBase<BlockT, LoopT> &LIB) {
assert(!isInvalid() && "Loop not in a valid state!");
#ifndef NDEBUG
- if (!Blocks.empty()) {
+ if (!getBlocks().empty()) {
auto SameHeader = LIB[getHeader()];
assert(contains(SameHeader) && getHeader() == SameHeader->getHeader() &&
"Incorrect LI specified for this loop!");
@@ -325,7 +325,7 @@ template <class BlockT, class LoopT>
void LoopBase<BlockT, LoopT>::verifyLoop() const {
assert(!isInvalid() && "Loop not in a valid state!");
#ifndef NDEBUG
- assert(!Blocks.empty() && "Loop header is missing");
+ assert(!getBlocks().empty() && "Loop header is missing");
// Setup for using a depth-first iterator to visit every block in the loop.
SmallVector<BlockT *, 8> ExitBBs;
@@ -371,7 +371,7 @@ void LoopBase<BlockT, LoopT>::verifyLoop() const {
if (VisitedBBs.size() != getNumBlocks()) {
dbgs() << "The following blocks are unreachable in the loop: ";
- for (auto *BB : Blocks) {
+ for (auto *BB : getBlocks()) {
if (!VisitedBBs.count(BB)) {
dbgs() << *BB << "\n";
}
@@ -455,12 +455,11 @@ void LoopBase<BlockT, LoopT>::print(raw_ostream &OS, bool Verbose,
/// this loop are mapped to this loop or a subloop. And all subloops within this
/// loop have their parent loop set to this loop or a subloop.
template <class BlockT, class LoopT>
-static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT *> Backedges,
- LoopInfoBase<BlockT, LoopT> *LI,
- const DomTreeBase<BlockT> &DomTree) {
+void LoopInfoBase<BlockT, LoopT>::discoverAndMapSubloop(
+ LoopT *L, BlockT *Header, ArrayRef<BlockT *> Backedges,
+ const DominatorTreeBase<BlockT, false> &DomTree) {
using InvBlockTraits = GraphTraits<Inverse<BlockT *>>;
- unsigned NumBlocks = 0;
unsigned NumSubloops = 0;
// Perform a backward CFG traversal using a worklist.
@@ -469,15 +468,14 @@ static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT *> Backedges,
BlockT *PredBB = ReverseCFGWorklist.back();
ReverseCFGWorklist.pop_back();
- LoopT *Subloop = LI->getLoopFor(PredBB);
+ LoopT *Subloop = getLoopFor(PredBB);
if (!Subloop) {
if (!DomTree.isReachableFromEntry(PredBB))
continue;
// This is an undiscovered block. Map it to the current loop.
- LI->changeLoopFor(PredBB, L);
- ++NumBlocks;
- if (PredBB == L->getHeader())
+ changeLoopFor(PredBB, L);
+ if (PredBB == Header)
continue;
// Push all block predecessors on the worklist.
ReverseCFGWorklist.insert(ReverseCFGWorklist.end(),
@@ -494,84 +492,31 @@ static void discoverAndMapSubloop(LoopT *L, ArrayRef<BlockT *> Backedges,
// Discover a subloop of this loop.
Subloop->setParentLoop(L);
++NumSubloops;
- NumBlocks += Subloop->getBlocksCapacity();
- PredBB = Subloop->getHeader();
+ PredBB = pendingHeader(Subloop);
// Continue traversal along predecessors that are not loop-back edges from
// within this subloop tree itself. Note that a predecessor may directly
// reach another subloop that is not yet discovered to be a subloop of
// this loop, which we must traverse.
for (const auto Pred : inverse_children<BlockT *>(PredBB)) {
- if (LI->getLoopFor(Pred) != Subloop)
+ if (getLoopFor(Pred) != Subloop)
ReverseCFGWorklist.push_back(Pred);
}
}
}
L->reserveSubLoops(NumSubloops);
- L->reserveBlocks(NumBlocks);
-}
-
-/// Populate all loop data in a stable order during a single forward DFS.
-template <class BlockT, class LoopT> class PopulateLoopsDFS {
- using BlockTraits = GraphTraits<BlockT *>;
- using SuccIterTy = typename BlockTraits::ChildIteratorType;
-
- LoopInfoBase<BlockT, LoopT> *LI;
-
-public:
- PopulateLoopsDFS(LoopInfoBase<BlockT, LoopT> *li) : LI(li) {}
-
- void traverse(BlockT *EntryBlock);
-
-protected:
- void insertIntoLoop(BlockT *Block);
-};
-
-/// Top-level driver for the forward DFS within the loop.
-template <class BlockT, class LoopT>
-void PopulateLoopsDFS<BlockT, LoopT>::traverse(BlockT *EntryBlock) {
- for (BlockT *BB : post_order(EntryBlock))
- insertIntoLoop(BB);
-}
-
-/// Add a single Block to its ancestor loops in PostOrder. If the block is a
-/// subloop header, add the subloop to its parent in PostOrder, then reverse the
-/// Block and Subloop vectors of the now complete subloop to achieve RPO.
-template <class BlockT, class LoopT>
-void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) {
- LoopT *Subloop = LI->getLoopFor(Block);
- if (Subloop && Block == Subloop->getHeader()) {
- // We reach this point once per subloop after processing all the blocks in
- // the subloop.
- if (!Subloop->isOutermost())
- Subloop->getParentLoop()->SubLoops.push_back(Subloop);
- else
- LI->addTopLevelLoop(Subloop);
-
- // For convenience, Blocks and Subloops are inserted in postorder. Reverse
- // the lists, except for the loop header, which is always at the beginning.
- Subloop->reverseBlock(1);
- std::reverse(Subloop->SubLoops.begin(), Subloop->SubLoops.end());
-
- Subloop = Subloop->getParentLoop();
- }
- for (; Subloop; Subloop = Subloop->getParentLoop())
- Subloop->addBlockEntry(Block);
}
/// Analyze LoopInfo discovers loops during a reverse preorder DominatorTree
/// traversal interleaved with backward CFG traversals within each subloop
/// (discoverAndMapSubloop). The backward traversal skips inner subloops, so
-/// this part of the algorithm is linear in the number of CFG edges. Subloop and
-/// Block vectors are then populated during a single forward CFG traversal
-/// (PopulateLoopDFS).
+/// this part of the algorithm is linear in the number of CFG edges.
///
-/// During the two CFG traversals each block is seen three times:
-/// 1) Discovered and mapped by a reverse CFG traversal.
-/// 2) Visited during a forward DFS CFG traversal.
-/// 3) Reverse-inserted in the loop in postorder following forward DFS.
-///
-/// The Block vectors are inclusive, so step 3 requires loop-depth number of
-/// insertions per block.
+/// The block lists are then built in one shared layout: a forward CFG postorder
+/// records the in-loop blocks, and a reverse walk (RPO) carves each loop a
+/// contiguous slice of its parent's, writing each block once to its innermost
+/// loop. Lists are header-first with each subloop's blocks contiguous, ordered
+/// by first appearance in RPO; SubLoops keep program order, TopLevelLoops
+/// reverse program order.
template <class BlockT, class LoopT>
void LoopInfoBase<BlockT, LoopT>::analyze(const DomTreeBase<BlockT> &DomTree) {
const DomTreeNodeBase<BlockT> *DomRoot = DomTree.getRootNode();
@@ -587,6 +532,7 @@ void LoopInfoBase<BlockT, LoopT>::analyze(const DomTreeBase<BlockT> &DomTree) {
for (const DomTreeNodeBase<BlockT> *Node : DomTree.nodes())
PreorderNodes[Node->getDFSNumIn()] = Node;
+ bool HasLoops = false;
for (const DomTreeNodeBase<BlockT> *DomNode : llvm::reverse(PreorderNodes)) {
BlockT *Header = DomNode->getBlock();
SmallVector<BlockT *, 4> Backedges;
@@ -600,14 +546,63 @@ void LoopInfoBase<BlockT, LoopT>::analyze(const DomTreeBase<BlockT> &DomTree) {
}
// Perform a backward CFG traversal to discover and map blocks in this loop.
if (!Backedges.empty()) {
- LoopT *L = AllocateLoop(Header);
- discoverAndMapSubloop(L, ArrayRef<BlockT *>(Backedges), this, DomTree);
+ HasLoops = true;
+ LoopT *L = allocateLoop(Header);
+ discoverAndMapSubloop(L, Header, Backedges, DomTree);
+ }
+ }
+ // Most functions have no loops; skip the layout construction.
+ if (!HasLoops)
+ return;
+
+ // Record each in-loop block with its innermost loop in forward CFG postorder,
+ // and build the loop list in PO.
+ SmallVector<std::pair<BlockT *, LoopT *>, 32> PO;
+ SmallVector<LoopT *, 4> LoopsPO;
+ PO.reserve(BBMap.size());
+ for (BlockT *BB : post_order(DomRoot->getBlock())) {
+ LoopT *L = lookupLoopFor(BB);
+ if (!L)
+ continue;
+ PO.emplace_back(BB, L);
+ ++L->BlockLen;
+ if (BB != pendingHeader(L))
+ continue;
+ LoopsPO.push_back(L);
+ if (LoopT *Parent = L->getParentLoop())
+ Parent->BlockLen += L->BlockLen;
+ else
+ TopLevelLoops.push_back(L);
+ }
+ // Headers are dominator-tree nodes, hence reachable and in the postorder.
+ assert(!LoopsPO.empty() && "discovered loops but found no header");
+
+ BlockLayoutLen = PO.size();
+ BlockLayout.reset(new BlockT *[BlockLayoutLen]);
+ BlockT **RootCursor = BlockLayout.get();
+ for (auto &[BB, L] : llvm::reverse(PO)) {
+ if (L->BlockCapacity == 0) {
+ // The first block of a L is its the header. Carve its slice from the
+ // parent (already visited)'s cursor.
+ if (LoopT *Parent = L->getParentLoop()) {
+ assert(Parent->BlockCapacity != 0 &&
+ "parent slice not carved before child");
+ L->BlockData = Parent->BlockData + Parent->BlockCapacity;
+ Parent->BlockCapacity += L->BlockLen;
+ Parent->SubLoops.push_back(L);
+ } else {
+ L->BlockData = RootCursor;
+ RootCursor += L->BlockLen;
+ }
}
+ // Each block lands once, at its innermost loop's cursor.
+ L->BlockData[L->BlockCapacity++] = BB;
}
- // Perform a single forward CFG traversal to populate block and subloop
- // vectors for all loops.
- PopulateLoopsDFS<BlockT, LoopT> DFS(this);
- DFS.traverse(DomRoot->getBlock());
+
+#ifndef NDEBUG
+ for (LoopT *L : LoopsPO)
+ assert(L->BlockCapacity == L->BlockLen && "layout slice not fully used");
+#endif
}
template <class BlockT, class LoopT>
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_outer_loop.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_outer_loop.ll
index 69ef9f3e530ad..aec401ece0b3a 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_outer_loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_outer_loop.ll
@@ -46,7 +46,7 @@ loop.inner.end:
; loops at depths 1 and 2, respectively.
define void @early_exit_in_outer_loop2() {
; CHECK-LABEL: Loop info for function 'early_exit_in_outer_loop2':
-; CHECK: Loop at depth 1 containing: %loop.outer<header>,%loop.middle,%loop.inner.found,%loop.inner.end,%loop.middle.end,%loop.outer.latch<latch>,%vector.ph,%vector.body,%vector.body.interim,%middle.block,%vector.early.exit
+; CHECK: Loop at depth 1 containing: %loop.outer<header>,%loop.middle,%loop.inner.end,%loop.inner.found,%loop.middle.end,%loop.outer.latch<latch>,%vector.ph,%vector.body,%vector.body.interim,%middle.block,%vector.early.exit
; CHECK: Loop at depth 2 containing: %loop.middle<header>,%loop.inner.end<latch><exiting>,%vector.ph,%vector.body<exiting>,%vector.body.interim,%middle.block
; CHECK: Loop at depth 3 containing: %vector.body<header><exiting>,%vector.body.interim<latch><exiting>
entry:
@@ -268,7 +268,7 @@ early.exit.leave:
; (one going to a middle loop, one going to the outer loop).
define i32 @multi_early_exit_two_different_loops(i1 %c, ptr dereferenceable(1024) %src) nofree {
; CHECK-LABEL: Loop info for function 'multi_early_exit_two_different_loops':
-; CHECK-NEXT: Loop at depth 1 containing: %outer.header<header>,%middle.header,%early.exit.outer,%early.exit.middle,%middle.latch,%outer.latch<latch>,%middle.latch.loopexit,%outer.latch.loopexit,%vector.ph,%vector.body,%vector.body.interim,%middle.block,%vector.early.exit.check,%vector.early.exit.1,%vector.early.exit.0
+; CHECK-NEXT: Loop at depth 1 containing: %outer.header<header>,%middle.header,%early.exit.middle,%middle.latch,%early.exit.outer,%outer.latch<latch>,%middle.latch.loopexit,%outer.latch.loopexit,%vector.ph,%vector.body,%vector.body.interim,%middle.block,%vector.early.exit.check,%vector.early.exit.1,%vector.early.exit.0
; CHECK-NEXT: Loop at depth 2 containing: %middle.header<header>,%early.exit.middle,%middle.latch<latch><exiting>,%middle.latch.loopexit,%vector.ph,%vector.body,%vector.body.interim,%middle.block,%vector.early.exit.check<exiting>,%vector.early.exit.0
; CHECK-NEXT: Loop at depth 3 containing: %vector.body<header><exiting>,%vector.body.interim<latch><exiting>
entry:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
index f55ec1bedc0b8..925b919c5a397 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
@@ -1134,10 +1134,10 @@ define i32 @test13a(ptr %ptr, i1 %cond, ptr %a.ptr, ptr %b.ptr) {
; CHECK: loop_b_inner_body.us:
; CHECK-NEXT: [[V4_US:%.*]] = load i1, ptr [[PTR]], align 1
; CHECK-NEXT: br i1 [[V4_US]], label [[LOOP_B_INNER_LATCH_US]], label [[LOOP_B_INNER_EXIT_US:%.*]]
-; CHECK: loop_b_inner_exit.us:
-; CHECK-NEXT: br label [[LOOP_LATCH_US:%.*]]
; CHECK: loop_b_inner_latch.us:
; CHECK-NEXT: br label [[LOOP_B_INNER_HEADER_US]]
+; CHECK: loop_b_inner_exit.us:
+; CHECK-NEXT: br label [[LOOP_LATCH_US:%.*]]
; CHECK: loop_a.us:
; CHECK-NEXT: [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
; CHECK-NEXT: br i1 [[V2_US]], label [[LOOP_EXIT_SPLIT_US:%.*]], label [[LOOP_LATCH_US]]
@@ -1426,10 +1426,10 @@ define i32 @test29(i32 %arg) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
; CHECK-NEXT: switch i32 [[ARG_FR]], label [[ENTRY_SPLIT:%.*]] [
-; CHECK-NEXT: i32 0, label [[ENTRY_SPLIT_US:%.*]]
-; CHECK-NEXT: i32 1, label [[ENTRY_SPLIT_US]]
-; CHECK-NEXT: i32 2, label [[ENTRY_SPLIT_US1:%.*]]
-; CHECK-NEXT: i32 3, label [[ENTRY_SPLIT]]
+; CHECK-NEXT: i32 0, label [[ENTRY_SPLIT_US:%.*]]
+; CHECK-NEXT: i32 1, label [[ENTRY_SPLIT_US]]
+; CHECK-NEXT: i32 2, label [[ENTRY_SPLIT_US1:%.*]]
+; CHECK-NEXT: i32 3, label [[ENTRY_SPLIT]]
; CHECK-NEXT: ]
; CHECK: entry.split.us:
; CHECK-NEXT: br label [[HEADER_US:%.*]]
@@ -1587,10 +1587,10 @@ define i32 @test30(i32 %arg) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
; CHECK-NEXT: switch i32 [[ARG_FR]], label [[ENTRY_SPLIT:%.*]] [
-; CHECK-NEXT: i32 -1, label [[ENTRY_SPLIT]]
-; CHECK-NEXT: i32 0, label [[ENTRY_SPLIT_US:%.*]]
-; CHECK-NEXT: i32 1, label [[ENTRY_SPLIT_US1:%.*]]
-; CHECK-NEXT: i32 2, label [[ENTRY_SPLIT_US1]]
+; CHECK-NEXT: i32 -1, label [[ENTRY_SPLIT]]
+; CHECK-NEXT: i32 0, label [[ENTRY_SPLIT_US:%.*]]
+; CHECK-NEXT: i32 1, label [[ENTRY_SPLIT_US1:%.*]]
+; CHECK-NEXT: i32 2, label [[ENTRY_SPLIT_US1]]
; CHECK-NEXT: ]
; CHECK: entry.split.us:
; CHECK-NEXT: br label [[HEADER_US:%.*]]
@@ -2259,9 +2259,9 @@ define void @hoist_inner_loop_switch(ptr %ptr) {
; CHECK-NEXT: [[V1:%.*]] = call i32 @cond.i32()
; CHECK-NEXT: [[V1_FR:%.*]] = freeze i32 [[V1]]
; CHECK-NEXT: switch i32 [[V1_FR]], label [[B_HEADER_SPLIT:%.*]] [
-; CHECK-NEXT: i32 1, label [[B_HEADER_SPLIT_US:%.*]]
-; CHECK-NEXT: i32 2, label [[B_HEADER_SPLIT_US]]
-; CHECK-NEXT: i32 3, label [[B_HEADER_SPLIT_US]]
+; CHECK-NEXT: i32 1, label [[B_HEADER_SPLIT_US:%.*]]
+; CHECK-NEXT: i32 2, label [[B_HEADER_SPLIT_US]]
+; CHECK-NEXT: i32 3, label [[B_HEADER_SPLIT_US]]
; CHECK-NEXT: ]
; CHECK: b.header.split.us:
; CHECK-NEXT: br label [[C_HEADER_US:%.*]]
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
index 0cb0e0d1cca2b..8fddaa1839576 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
@@ -2116,12 +2116,12 @@ loop_latch:
; CHECK-NEXT: %[[V:.*]] = load i1, ptr %ptr
; CHECK-NEXT: br i1 %[[V]], label %loop_b_inner_latch.us, label %loop_b_inner_exit.us
;
-; CHECK: loop_b_inner_exit.us:
-; CHECK-NEXT: br label %loop_latch.us
-;
; CHECK: loop_b_inner_latch.us:
; CHECK-NEXT: br label %loop_b_inner_header.us
;
+; CHECK: loop_b_inner_exit.us:
+; CHECK-NEXT: br label %loop_latch.us
+;
; CHECK: loop_a.us:
; CHECK-NEXT: %[[V:.*]] = load i1, ptr %ptr
; CHECK-NEXT: br i1 %[[V]], label %loop_exit.split.us, label %loop_latch.us
diff --git a/mlir/include/mlir/Analysis/CFGLoopInfo.h b/mlir/include/mlir/Analysis/CFGLoopInfo.h
index 15ac0d71cb94d..7060a9df7de2d 100644
--- a/mlir/include/mlir/Analysis/CFGLoopInfo.h
+++ b/mlir/include/mlir/Analysis/CFGLoopInfo.h
@@ -36,7 +36,7 @@ namespace mlir {
/// class provides accessors to the loop analysis.
class CFGLoop : public llvm::LoopBase<mlir::Block, mlir::CFGLoop> {
private:
- explicit CFGLoop(mlir::Block *block);
+ CFGLoop() = default;
friend class llvm::LoopBase<mlir::Block, CFGLoop>;
friend class llvm::LoopInfoBase<mlir::Block, CFGLoop>;
diff --git a/mlir/lib/Analysis/CFGLoopInfo.cpp b/mlir/lib/Analysis/CFGLoopInfo.cpp
index 9fe220344dcea..e03b664f9c72e 100644
--- a/mlir/lib/Analysis/CFGLoopInfo.cpp
+++ b/mlir/lib/Analysis/CFGLoopInfo.cpp
@@ -16,9 +16,6 @@ template class llvm::LoopInfoBase<mlir::Block, mlir::CFGLoop>;
using namespace mlir;
-CFGLoop::CFGLoop(mlir::Block *block)
- : llvm::LoopBase<mlir::Block, CFGLoop>(block) {}
-
CFGLoopInfo::CFGLoopInfo(
const llvm::DominatorTreeBase<mlir::Block, false> &domTree) {
analyze(domTree);
More information about the Mlir-commits
mailing list