[llvm] [CycleInfo] Move representation-dependent queries to GenericCycleInfo. NFC (PR #209665)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 19:49:35 PDT 2026
https://github.com/MaskRay created https://github.com/llvm/llvm-project/pull/209665
With the Euler tour representation (#208614), blocks live in
GenericCycleInfo::BlockLayout and GenericCycle keeps a `CI` back-pointer just
so the out-of-line contains(BlockT *) and blocks() can reach it.
Adopt the design suggested by @aengelke, move the
representation-dependent queries to GenericCycleInfo, taking the cycle
as an argument (contains, getBlocks, getExitBlocks, getExitingBlocks,
getCyclePreheader, getCyclePredecessor, verifyCycle, verifyCycleNest,
and per-cycle print), and delete GenericCycle::CI.
- GenericCycleInfo's move operations become defaulted, dropping the
CI re-pointing walk, and sizeof(GenericCycle) shrinks by a pointer.
- isCycleInvariant gains a MachineCycleInfo parameter, and
GenericUniformityInfo gains getCycleInfo() for callers that only hold the
uniformity result.
Aided by Claude Fable 5
>From 178f27b44b22d9e8ac928c62bcfedcb1c7ed87ac Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Tue, 14 Jul 2026 19:49:23 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.5-bogner
---
llvm/include/llvm/ADT/GenericCycleImpl.h | 188 +++++++-----------
llvm/include/llvm/ADT/GenericCycleInfo.h | 158 +++++++--------
llvm/include/llvm/ADT/GenericUniformityImpl.h | 81 ++++----
llvm/include/llvm/ADT/GenericUniformityInfo.h | 3 +
.../llvm/CodeGen/MachineCycleAnalysis.h | 7 +-
.../llvm/IR/GenericConvergenceVerifierImpl.h | 4 +-
llvm/lib/Analysis/CFG.cpp | 2 +-
llvm/lib/Analysis/UniformityAnalysis.cpp | 4 +-
llvm/lib/CodeGen/MachineCycleAnalysis.cpp | 7 +-
llvm/lib/CodeGen/MachineSink.cpp | 8 +-
.../lib/CodeGen/MachineUniformityAnalysis.cpp | 4 +-
.../AMDGPUGlobalISelDivergenceLowering.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +-
.../Transforms/IPO/AttributorAttributes.cpp | 2 +-
llvm/lib/Transforms/Utils/FixIrreducible.cpp | 22 +-
15 files changed, 227 insertions(+), 268 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index dfeb479d764e5..02e6770f0aedd 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -34,60 +34,39 @@
namespace llvm {
template <typename ContextT>
-bool GenericCycle<ContextT>::contains(const GenericCycle *C) const {
- // Containment check using the Euler tour representation.
- return C && IdxBegin <= C->IdxBegin && C->IdxEnd <= IdxEnd;
-}
-
-template <typename ContextT>
-bool GenericCycle<ContextT>::contains(const BlockT *Block) const {
- return contains(CI->getCycle(Block));
-}
-
-template <typename ContextT>
-auto GenericCycle<ContextT>::block_begin() const -> const_block_iterator {
- return CI->BlockLayout.begin() + IdxBegin;
-}
-
-template <typename ContextT>
-auto GenericCycle<ContextT>::block_end() const -> const_block_iterator {
- return CI->BlockLayout.begin() + IdxEnd;
-}
-
-template <typename ContextT>
-void GenericCycle<ContextT>::getExitBlocks(
- SmallVectorImpl<BlockT *> &TmpStorage) const {
- if (!ExitBlocksCache.empty()) {
- TmpStorage.append(ExitBlocksCache.begin(), ExitBlocksCache.end());
+void GenericCycleInfo<ContextT>::getExitBlocks(
+ const CycleT &C, SmallVectorImpl<BlockT *> &TmpStorage) const {
+ if (!C.ExitBlocksCache.empty()) {
+ TmpStorage.append(C.ExitBlocksCache.begin(), C.ExitBlocksCache.end());
return;
}
size_t NumExitBlocks = 0;
- for (BlockT *Block : blocks()) {
- llvm::append_range(ExitBlocksCache, successors(Block));
+ for (BlockT *Block : getBlocks(C)) {
+ llvm::append_range(C.ExitBlocksCache, successors(Block));
- for (size_t Idx = NumExitBlocks, End = ExitBlocksCache.size(); Idx < End;
+ for (size_t Idx = NumExitBlocks, End = C.ExitBlocksCache.size(); Idx < End;
++Idx) {
- BlockT *Succ = ExitBlocksCache[Idx];
- if (!contains(Succ)) {
- auto ExitEndIt = ExitBlocksCache.begin() + NumExitBlocks;
- if (std::find(ExitBlocksCache.begin(), ExitEndIt, Succ) == ExitEndIt)
- ExitBlocksCache[NumExitBlocks++] = Succ;
+ BlockT *Succ = C.ExitBlocksCache[Idx];
+ if (!contains(&C, Succ)) {
+ auto ExitEndIt = C.ExitBlocksCache.begin() + NumExitBlocks;
+ if (std::find(C.ExitBlocksCache.begin(), ExitEndIt, Succ) == ExitEndIt)
+ C.ExitBlocksCache[NumExitBlocks++] = Succ;
}
}
- ExitBlocksCache.resize(NumExitBlocks);
+ C.ExitBlocksCache.resize(NumExitBlocks);
}
- TmpStorage.append(ExitBlocksCache.begin(), ExitBlocksCache.end());
+ TmpStorage.append(C.ExitBlocksCache.begin(), C.ExitBlocksCache.end());
}
template <typename ContextT>
-void GenericCycle<ContextT>::getExitingBlocks(
- SmallVectorImpl<BlockT *> &TmpStorage) const {
- for (BlockT *Block : blocks()) {
+void GenericCycleInfo<ContextT>::getExitingBlocks(
+ const CycleT &C, SmallVectorImpl<BlockT *> &TmpStorage) const {
+ for (BlockT *Block : getBlocks(C)) {
for (BlockT *Succ : successors(Block)) {
- if (!contains(Succ)) {
+ if (!contains(&C, Succ)) {
TmpStorage.push_back(Block);
break;
}
@@ -96,12 +75,13 @@ void GenericCycle<ContextT>::getExitingBlocks(
}
template <typename ContextT>
-auto GenericCycle<ContextT>::getCyclePreheader() const -> BlockT * {
- BlockT *Predecessor = getCyclePredecessor();
+auto GenericCycleInfo<ContextT>::getCyclePreheader(const CycleT &C) const
+ -> BlockT * {
+ BlockT *Predecessor = getCyclePredecessor(C);
if (!Predecessor)
return nullptr;
- assert(isReducible() && "Cycle Predecessor must be in a reducible cycle!");
+ assert(C.isReducible() && "Cycle Predecessor must be in a reducible cycle!");
if (succ_size(Predecessor) != 1)
return nullptr;
@@ -114,16 +94,17 @@ auto GenericCycle<ContextT>::getCyclePreheader() const -> BlockT * {
}
template <typename ContextT>
-auto GenericCycle<ContextT>::getCyclePredecessor() const -> BlockT * {
- if (!isReducible())
+auto GenericCycleInfo<ContextT>::getCyclePredecessor(const CycleT &C) const
+ -> BlockT * {
+ if (!C.isReducible())
return nullptr;
BlockT *Out = nullptr;
// Loop over the predecessors of the header node...
- BlockT *Header = getHeader();
+ BlockT *Header = C.getHeader();
for (const auto Pred : predecessors(Header)) {
- if (!contains(Pred)) {
+ if (!contains(&C, Pred)) {
if (Out && Out != Pred)
return nullptr;
Out = Pred;
@@ -133,25 +114,25 @@ auto GenericCycle<ContextT>::getCyclePredecessor() const -> BlockT * {
return Out;
}
-/// \brief Verify that this is actually a well-formed cycle in the CFG.
-template <typename ContextT> void GenericCycle<ContextT>::verifyCycle() const {
+template <typename ContextT>
+void GenericCycleInfo<ContextT>::verifyCycle(const CycleT &C) const {
#ifndef NDEBUG
- assert(getNumBlocks() != 0 && "Cycle cannot be empty.");
+ assert(C.getNumBlocks() != 0 && "Cycle cannot be empty.");
DenseSet<BlockT *> Blocks;
- for (BlockT *BB : blocks()) {
+ for (BlockT *BB : getBlocks(C)) {
assert(Blocks.insert(BB).second); // duplicates in block list?
}
- assert(!Entries.empty() && "Cycle must have one or more entries.");
+ assert(!C.Entries.empty() && "Cycle must have one or more entries.");
DenseSet<BlockT *> Entries;
- for (BlockT *Entry : entries()) {
+ for (BlockT *Entry : C.entries()) {
assert(Entries.insert(Entry).second); // duplicate entry?
- assert(contains(Entry));
+ assert(contains(&C, Entry));
}
// Setup for using a depth-first iterator to visit every block in the cycle.
SmallVector<BlockT *, 8> ExitBBs;
- getExitBlocks(ExitBBs);
+ getExitBlocks(C, ExitBBs);
df_iterator_default_set<BlockT *> VisitSet;
VisitSet.insert(ExitBBs.begin(), ExitBBs.end());
@@ -159,18 +140,18 @@ template <typename ContextT> void GenericCycle<ContextT>::verifyCycle() const {
SmallPtrSet<BlockT *, 8> VisitedBBs;
// Check the individual blocks.
- for (BlockT *BB : depth_first_ext(getHeader(), VisitSet)) {
+ for (BlockT *BB : depth_first_ext(C.getHeader(), VisitSet)) {
assert(llvm::any_of(llvm::children<BlockT *>(BB),
- [&](BlockT *B) { return contains(B); }) &&
+ [&](BlockT *B) { return contains(&C, B); }) &&
"Cycle block has no in-cycle successors!");
assert(llvm::any_of(llvm::inverse_children<BlockT *>(BB),
- [&](BlockT *B) { return contains(B); }) &&
+ [&](BlockT *B) { return contains(&C, B); }) &&
"Cycle block has no in-cycle predecessors!");
DenseSet<BlockT *> OutsideCyclePreds;
for (BlockT *B : llvm::inverse_children<BlockT *>(BB))
- if (!contains(B))
+ if (!contains(&C, B))
OutsideCyclePreds.insert(B);
if (Entries.contains(BB)) {
@@ -184,13 +165,13 @@ template <typename ContextT> void GenericCycle<ContextT>::verifyCycle() const {
assert(!OutsideCyclePreds.contains(CB) &&
"Non-entry block reachable from outside!");
}
- assert(BB != &getHeader()->getParent()->front() &&
+ assert(BB != &C.getHeader()->getParent()->front() &&
"Cycle contains function entry block!");
VisitedBBs.insert(BB);
}
- if (VisitedBBs.size() != getNumBlocks()) {
+ if (VisitedBBs.size() != C.getNumBlocks()) {
dbgs() << "The following blocks are unreachable in the cycle:\n ";
ListSeparator LS;
for (auto *BB : Blocks) {
@@ -203,34 +184,31 @@ template <typename ContextT> void GenericCycle<ContextT>::verifyCycle() const {
llvm_unreachable("Unreachable block in cycle");
}
- verifyCycleNest();
+ verifyCycleNest(C);
#endif
}
-/// \brief Verify the parent-child relations of this cycle.
-///
-/// Note that this does \em not check that cycle is really a cycle in the CFG.
template <typename ContextT>
-void GenericCycle<ContextT>::verifyCycleNest() const {
+void GenericCycleInfo<ContextT>::verifyCycleNest(const CycleT &C) const {
#ifndef NDEBUG
// Check the subcycles.
- for (GenericCycle *Child : children()) {
+ for (CycleT *Child : C.children()) {
// Each block in each subcycle should be contained within this cycle.
- for (BlockT *BB : Child->blocks()) {
- assert(contains(BB) &&
+ for (BlockT *BB : getBlocks(*Child)) {
+ assert(contains(&C, BB) &&
"Cycle does not contain all the blocks of a subcycle!");
}
- assert(Child->Depth == Depth + 1);
+ assert(Child->Depth == C.Depth + 1);
}
// Check the parent cycle pointer.
- if (ParentCycle) {
- assert(is_contained(ParentCycle->children(), this) &&
+ if (C.ParentCycle) {
+ assert(is_contained(C.ParentCycle->children(), &C) &&
"Cycle is not a subcycle of its parent!");
- assert(ParentCycle->TopLevelCycle == TopLevelCycle &&
+ assert(C.ParentCycle->TopLevelCycle == C.TopLevelCycle &&
"Top level cycle of parent cycle must be the same");
} else {
- assert(TopLevelCycle == this &&
+ assert(C.TopLevelCycle == &C &&
"Cycle without parent must be top-level cycle");
}
#endif
@@ -289,13 +267,6 @@ template <typename ContextT> class GenericCycleInfoCompute {
void dfs(FunctionT *F, BlockT *EntryBlock);
};
-template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getTopLevelParentCycle(
- const BlockT *Block) const -> CycleT * {
- CycleT *Cycle = getCycle(Block);
- return Cycle ? Cycle->TopLevelCycle : nullptr;
-}
-
template <typename ContextT>
void GenericCycleInfo<ContextT>::moveTopLevelCycleToNewParent(CycleT *NewParent,
CycleT *Child) {
@@ -320,14 +291,6 @@ void GenericCycleInfo<ContextT>::moveTopLevelCycleToNewParent(CycleT *NewParent,
// range-dependent query is used.
}
-template <typename ContextT>
-void GenericCycleInfo<ContextT>::verifyBlockNumberEpoch(
- const FunctionT *Fn) const {
- assert(BlockNumberEpoch ==
- GraphTraits<const FunctionT *>::getNumberEpoch(Fn) &&
- "CycleInfo used with outdated block number epoch");
-}
-
template <typename ContextT>
void GenericCycleInfo<ContextT>::addToBlockMap(BlockT *Block, CycleT *Cycle) {
// The caller should ensure that BlockMap is large enough.
@@ -438,7 +401,6 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
LLVM_DEBUG(errs() << "Found cycle for header: "
<< Info.Context.print(HeaderCandidate) << "\n");
std::unique_ptr<CycleT> NewCycle = std::make_unique<CycleT>();
- NewCycle->CI = &Info;
NewCycle->appendEntry(HeaderCandidate);
Info.addToBlockMap(HeaderCandidate, NewCycle.get());
// The header is this cycle's first own block. Until layoutBlocks runs,
@@ -624,18 +586,6 @@ void GenericCycleInfo<ContextT>::splitCriticalEdge(BlockT *Pred, BlockT *Succ,
verifyCycleNest();
}
-/// \brief Find the innermost cycle containing a given block.
-///
-/// \returns the innermost cycle containing \p Block or nullptr if
-/// it is not contained in any cycle.
-template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getCycle(const BlockT *Block) const
- -> CycleT * {
- verifyBlockNumberEpoch(Block->getParent());
- unsigned Number = GraphTraits<const BlockT *>::getNumber(Block);
- return Number < BlockMap.size() ? BlockMap[Number] : nullptr;
-}
-
/// \brief Find the innermost cycle containing both given cycles.
///
/// \returns the innermost cycle containing both \p A and \p B
@@ -676,18 +626,6 @@ auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(BlockT *A,
return getSmallestCommonCycle(getCycle(A), getCycle(B));
}
-/// \brief get the depth for the cycle which containing a given block.
-///
-/// \returns the depth for the innermost cycle containing \p Block or 0 if it is
-/// not contained in any cycle.
-template <typename ContextT>
-unsigned GenericCycleInfo<ContextT>::getCycleDepth(const BlockT *Block) const {
- CycleT *Cycle = getCycle(Block);
- if (!Cycle)
- return 0;
- return Cycle->getDepth();
-}
-
/// \brief Verify the internal consistency of the cycle tree.
///
/// Note that this does \em not check that cycles are really cycles in the CFG,
@@ -702,11 +640,11 @@ void GenericCycleInfo<ContextT>::verifyCycleNest(bool VerifyFull) const {
BlockT *Header = Cycle->getHeader();
assert(CycleHeaders.insert(Header).second);
if (VerifyFull)
- Cycle->verifyCycle();
+ verifyCycle(*Cycle);
else
- Cycle->verifyCycleNest();
+ verifyCycleNest(*Cycle);
// Check the block map entries for blocks contained in this cycle.
- for (BlockT *BB : Cycle->blocks()) {
+ for (BlockT *BB : getBlocks(*Cycle)) {
CycleT *CycleInBlockMap = getCycle(BB);
assert(CycleInBlockMap != nullptr);
assert(Cycle->contains(CycleInBlockMap));
@@ -729,11 +667,27 @@ void GenericCycleInfo<ContextT>::print(raw_ostream &Out) const {
for (unsigned I = 0; I < Cycle->Depth; ++I)
Out << " ";
- Out << Cycle->print(Context) << '\n';
+ Out << print(Cycle) << '\n';
}
}
}
+/// \brief Print a single cycle: its depth, entries, and remaining blocks.
+template <typename ContextT>
+Printable GenericCycleInfo<ContextT>::print(const CycleT *Cycle) const {
+ return Printable([this, Cycle](raw_ostream &Out) {
+ Out << "depth=" << Cycle->Depth << ": entries("
+ << Cycle->printEntries(Context) << ')';
+
+ for (auto *Block : getBlocks(*Cycle)) {
+ if (Cycle->isEntry(Block))
+ continue;
+
+ Out << ' ' << Context.print(Block);
+ }
+ });
+}
+
} // namespace llvm
#undef DEBUG_TYPE
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index c3093d4cf83e5..d3ffafc7d0d39 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -82,9 +82,6 @@ template <typename ContextT> class GenericCycle {
/// always have the same depth.
unsigned Depth = 0;
- /// The cycle info that owns this cycle. Used by contains(BlockT*).
- const GenericCycleInfo<ContextT> *CI = nullptr;
-
/// Cache for the results of GetExitBlocks
mutable SmallVector<BlockT *, 4> ExitBlocksCache;
@@ -130,46 +127,24 @@ template <typename ContextT> class GenericCycle {
}
/// \brief Replace all entries with \p Block as single entry.
+ /// \p Block must be contained in the cycle.
void setSingleEntry(BlockT *Block) {
- assert(contains(Block));
Entries.clear();
Entries.push_back(Block);
clearCache();
}
- /// \brief Return whether \p Block is contained in the cycle. O(1).
- bool contains(const BlockT *Block) const;
-
/// \brief Returns true iff this cycle contains \p C. O(1). Non-strict, i.e.
/// returns true if C is the same cycle.
- bool contains(const GenericCycle *C) const;
+ bool contains(const GenericCycle *C) const {
+ return C && IdxBegin <= C->IdxBegin && C->IdxEnd <= IdxEnd;
+ }
const GenericCycle *getParentCycle() const { return ParentCycle; }
GenericCycle *getParentCycle() { return ParentCycle; }
unsigned getDepth() const { return Depth; }
- /// Return all of the successor blocks of this cycle.
- ///
- /// These are the blocks _outside of the current cycle_ which are
- /// branched to.
- void getExitBlocks(SmallVectorImpl<BlockT *> &TmpStorage) const;
-
- /// Return all blocks of this cycle that have successor outside of this cycle.
- /// These blocks have cycle exit branch.
- void getExitingBlocks(SmallVectorImpl<BlockT *> &TmpStorage) const;
-
- /// Return the preheader block for this cycle. Pre-header is well-defined for
- /// reducible cycle in docs/LoopTerminology.md as: the only one entering
- /// block and its only edge is to the entry block. Return null for irreducible
- /// cycles.
- BlockT *getCyclePreheader() const;
-
- /// If the cycle has exactly one entry with exactly one predecessor, return
- /// it, otherwise return nullptr.
- BlockT *getCyclePredecessor() const;
-
- void verifyCycle() const;
- void verifyCycleNest() const;
+ size_t getNumBlocks() const { return IdxEnd - IdxBegin; }
/// Iteration over child cycles.
//@{
@@ -200,19 +175,6 @@ template <typename ContextT> class GenericCycle {
}
//@}
- /// Iteration over blocks in the cycle (including entry blocks).
- //@{
- using const_block_iterator =
- typename SmallVector<BlockT *, 8>::const_iterator;
-
- const_block_iterator block_begin() const;
- const_block_iterator block_end() const;
- size_t getNumBlocks() const { return IdxEnd - IdxBegin; }
- iterator_range<const_block_iterator> blocks() const {
- return llvm::make_range(block_begin(), block_end());
- }
- //@}
-
/// Iteration over entry blocks.
//@{
using const_entry_iterator =
@@ -236,19 +198,6 @@ template <typename ContextT> class GenericCycle {
Out << LS << Ctx.print(Entry);
});
}
-
- Printable print(const ContextT &Ctx) const {
- return Printable([this, &Ctx](raw_ostream &Out) {
- Out << "depth=" << Depth << ": entries(" << printEntries(Ctx) << ')';
-
- for (auto *Block : blocks()) {
- if (isEntry(Block))
- continue;
-
- Out << ' ' << Ctx.print(Block);
- }
- });
- }
};
/// \brief Cycle information for a function.
@@ -257,7 +206,6 @@ template <typename ContextT> class GenericCycleInfo {
using BlockT = typename ContextT::BlockT;
using CycleT = GenericCycle<ContextT>;
using FunctionT = typename ContextT::FunctionT;
- template <typename> friend class GenericCycle;
template <typename> friend class GenericCycleInfoCompute;
private:
@@ -283,7 +231,11 @@ template <typename ContextT> class GenericCycleInfo {
/// the subtree.
void moveTopLevelCycleToNewParent(CycleT *NewParent, CycleT *Child);
- void verifyBlockNumberEpoch(const FunctionT *Fn) const;
+ void verifyBlockNumberEpoch(const FunctionT *Fn) const {
+ assert(BlockNumberEpoch ==
+ GraphTraits<const FunctionT *>::getNumberEpoch(Fn) &&
+ "CycleInfo used with outdated block number epoch");
+ }
void addToBlockMap(BlockT *Block, CycleT *Cycle);
/// Build BlockLayout and every cycle's [IdxBegin, IdxEnd) slice
@@ -292,29 +244,8 @@ template <typename ContextT> class GenericCycleInfo {
public:
GenericCycleInfo() = default;
- GenericCycleInfo(GenericCycleInfo &&Other) { *this = std::move(Other); }
- GenericCycleInfo &operator=(GenericCycleInfo &&Other) {
- if (this == &Other)
- return *this;
- Context = std::move(Other.Context);
- BlockNumberEpoch = Other.BlockNumberEpoch;
- BlockMap = std::move(Other.BlockMap);
- BlockLayout = std::move(Other.BlockLayout);
- TopLevelCycles = std::move(Other.TopLevelCycles);
- // The moved cycles carry a back-reference to their owning info (used by
- // GenericCycle::contains(BlockT*) and blocks()); re-point it at this
- // object.
- SmallVector<CycleT *, 8> Worklist;
- for (auto &TLC : TopLevelCycles)
- Worklist.push_back(TLC.get());
- while (!Worklist.empty()) {
- CycleT *C = Worklist.pop_back_val();
- C->CI = this;
- for (auto &Child : C->Children)
- Worklist.push_back(Child.get());
- }
- return *this;
- }
+ GenericCycleInfo(GenericCycleInfo &&) = default;
+ GenericCycleInfo &operator=(GenericCycleInfo &&) = default;
void clear();
void compute(FunctionT &F);
@@ -323,11 +254,68 @@ template <typename ContextT> class GenericCycleInfo {
const FunctionT *getFunction() const { return Context.getFunction(); }
const ContextT &getSSAContext() const { return Context; }
- CycleT *getCycle(const BlockT *Block) const;
+ /// \brief Find the innermost cycle containing \p Block.
+ ///
+ /// \returns the innermost cycle containing \p Block or nullptr if
+ /// it is not contained in any cycle.
+ CycleT *getCycle(const BlockT *Block) const {
+ verifyBlockNumberEpoch(Block->getParent());
+ unsigned Number = GraphTraits<const BlockT *>::getNumber(Block);
+ return Number < BlockMap.size() ? BlockMap[Number] : nullptr;
+ }
+
+ /// \brief Return whether \p Block is contained in \p C. O(1).
+ bool contains(const CycleT *C, const BlockT *Block) const {
+ return C && C->contains(getCycle(Block));
+ }
+
+ /// \brief Return the blocks of \p C, including those of nested cycles.
+ ArrayRef<BlockT *> getBlocks(const CycleT &C) const {
+ return ArrayRef<BlockT *>(BlockLayout.begin() + C.IdxBegin,
+ BlockLayout.begin() + C.IdxEnd);
+ }
+
CycleT *getSmallestCommonCycle(CycleT *A, CycleT *B) const;
CycleT *getSmallestCommonCycle(BlockT *A, BlockT *B) const;
- unsigned getCycleDepth(const BlockT *Block) const;
- CycleT *getTopLevelParentCycle(const BlockT *Block) const;
+
+ /// \brief Return the depth of the innermost cycle containing \p Block, or 0
+ /// if it is not contained in any cycle.
+ unsigned getCycleDepth(const BlockT *Block) const {
+ CycleT *Cycle = getCycle(Block);
+ return Cycle ? Cycle->getDepth() : 0;
+ }
+
+ CycleT *getTopLevelParentCycle(const BlockT *Block) const {
+ CycleT *Cycle = getCycle(Block);
+ return Cycle ? Cycle->TopLevelCycle : nullptr;
+ }
+
+ /// Return all of the successor blocks of \p C: the blocks outside of \p C
+ /// which are branched to from within it.
+ void getExitBlocks(const CycleT &C,
+ SmallVectorImpl<BlockT *> &TmpStorage) const;
+
+ /// Return all blocks of \p C that have a successor outside of \p C.
+ void getExitingBlocks(const CycleT &C,
+ SmallVectorImpl<BlockT *> &TmpStorage) const;
+
+ /// Return the preheader block for \p C. Pre-header is well-defined for
+ /// reducible cycle in docs/LoopTerminology.md as: the only one entering
+ /// block and its only edge is to the entry block. Return null for
+ /// irreducible cycles.
+ BlockT *getCyclePreheader(const CycleT &C) const;
+
+ /// If \p C has exactly one entry with exactly one predecessor, return it,
+ /// otherwise return nullptr.
+ BlockT *getCyclePredecessor(const CycleT &C) const;
+
+ /// Verify that \p C is actually a well-formed cycle in the CFG.
+ void verifyCycle(const CycleT &C) const;
+
+ /// Verify the parent-child relations of \p C.
+ ///
+ /// Note that this does \em not check that \p C is really a cycle in the CFG.
+ void verifyCycleNest(const CycleT &C) const;
/// Assumes that \p Cycle is the innermost cycle containing \p Block.
/// \p Block will be appended to \p Cycle and all of its parent cycles.
@@ -341,7 +329,7 @@ template <typename ContextT> class GenericCycleInfo {
void verify() const;
void print(raw_ostream &Out) const;
void dump() const { print(dbgs()); }
- Printable print(const CycleT *Cycle) { return Cycle->print(Context); }
+ Printable print(const CycleT *Cycle) const;
//@}
/// Iteration over top-level cycles.
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 10a58d6574de5..1c329e700c022 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -357,6 +357,8 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
const FunctionT &getFunction() const { return F; }
+ const CycleInfoT &getCycleInfo() const { return CI; }
+
/// \brief Mark \p UniVal as a value that is always uniform.
void addUniformOverride(const InstructionT &Instr);
@@ -656,7 +658,7 @@ template <typename ContextT> class DivergencePropagator {
// Bootstrap with branch targets
for (const auto *SuccBlock : successors(&DivTermBlock)) {
- if (DivTermCycle && !DivTermCycle->contains(SuccBlock)) {
+ if (DivTermCycle && !CI.contains(DivTermCycle, SuccBlock)) {
// If DivTerm exits the cycle immediately, computeJoin() might
// not reach SuccBlock with a different label. We need to
// check for this exit now.
@@ -681,7 +683,7 @@ template <typename ContextT> class DivergencePropagator {
// If no irreducible cycle, stop if freshLable.count() = 1 and Block
// is the IPD. If it is in any irreducible cycle, continue propagation.
if (FreshLabels.count() == 1 &&
- (!IrreducibleAncestor || !IrreducibleAncestor->contains(Block)))
+ (!IrreducibleAncestor || !CI.contains(IrreducibleAncestor, Block)))
break;
LLVM_DEBUG(dbgs() << "Current labels:\n"; printDefs(dbgs()));
@@ -716,10 +718,10 @@ template <typename ContextT> class DivergencePropagator {
<< '\n');
if (CyclePOT.isReducibleCycleHeader(Block)) {
const auto *BlockCycle = CI.getCycle(Block);
- LLVM_DEBUG(dbgs() << BlockCycle->print(Context) << '\n');
+ LLVM_DEBUG(dbgs() << CI.print(BlockCycle) << '\n');
SmallVector<BlockT *, 4> BlockCycleExits;
- BlockCycle->getExitBlocks(BlockCycleExits);
- bool BranchIsInside = BlockCycle->contains(&DivTermBlock);
+ CI.getExitBlocks(*BlockCycle, BlockCycleExits);
+ bool BranchIsInside = CI.contains(BlockCycle, &DivTermBlock);
for (auto *BlockCycleExit : BlockCycleExits) {
if (BranchIsInside)
visitCycleExitEdge(*BlockCycleExit, *Label);
@@ -745,7 +747,7 @@ template <typename ContextT> class DivergencePropagator {
continue;
}
SmallVector<BlockT *> Exits;
- Cycle->getExitBlocks(Exits);
+ CI.getExitBlocks(*Cycle, Exits);
auto *Header = Cycle->getHeader();
auto *HeaderLabel = BlockLabels[Header];
for (const auto *Exit : Exits) {
@@ -876,7 +878,7 @@ template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
const CycleT &DefCycle) {
SmallVector<BlockT *> Exits;
- DefCycle.getExitBlocks(Exits);
+ CI.getExitBlocks(DefCycle, Exits);
for (auto *Exit : Exits) {
for (auto &Phi : Exit->phis()) {
if (usesValueFromCycle(Phi, DefCycle)) {
@@ -885,7 +887,7 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
}
}
- for (auto *BB : DefCycle.blocks()) {
+ for (auto *BB : CI.getBlocks(DefCycle)) {
if (!llvm::any_of(Exits,
[&](BlockT *Exit) { return DT.dominates(BB, Exit); }))
continue;
@@ -981,18 +983,19 @@ bool insertIfNotContained(SmallVector<CycleT *> &Cycles, CycleT *Candidate) {
/// If two paths that diverged outside an irreducible cycle join
/// inside that cycle, then that whole cycle is assumed to be
/// divergent. This does not apply if the cycle is reducible.
-template <typename CycleT, typename BlockT>
-const CycleT *getExtDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
+template <typename CycleInfoT, typename CycleT, typename BlockT>
+const CycleT *getExtDivCycle(const CycleInfoT &CI, const CycleT *Cycle,
+ const BlockT *DivTermBlock,
const BlockT *JoinBlock) {
assert(Cycle);
- assert(Cycle->contains(JoinBlock));
+ assert(CI.contains(Cycle, JoinBlock));
- if (Cycle->contains(DivTermBlock))
+ if (CI.contains(Cycle, DivTermBlock))
return nullptr;
const auto *OriginalCycle = Cycle;
const auto *Parent = Cycle->getParentCycle();
- while (Parent && !Parent->contains(DivTermBlock)) {
+ while (Parent && !CI.contains(Parent, DivTermBlock)) {
Cycle = Parent;
Parent = Cycle->getParentCycle();
}
@@ -1016,9 +1019,10 @@ const CycleT *getExtDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
///
/// This checks the "diverged entry" criterion defined in the
/// docs/ConvergenceAnalysis.html.
-template <typename ContextT, typename CycleT, typename BlockT,
- typename DominatorTreeT>
-const CycleT *getIntDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
+template <typename ContextT, typename CycleInfoT, typename CycleT,
+ typename BlockT, typename DominatorTreeT>
+const CycleT *getIntDivCycle(const CycleInfoT &CI, const CycleT *Cycle,
+ const BlockT *DivTermBlock,
const BlockT *JoinBlock, const DominatorTreeT &DT,
ContextT &Context) {
LLVM_DEBUG(dbgs() << "examine join " << Context.print(JoinBlock)
@@ -1028,8 +1032,8 @@ const CycleT *getIntDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
return nullptr;
// Find the smallest common cycle, if one exists.
- assert(Cycle && Cycle->contains(JoinBlock));
- while (Cycle && !Cycle->contains(DivTermBlock)) {
+ assert(Cycle && CI.contains(Cycle, JoinBlock));
+ while (Cycle && !CI.contains(Cycle, DivTermBlock)) {
Cycle = Cycle->getParentCycle();
}
if (!Cycle || Cycle->isReducible())
@@ -1053,21 +1057,22 @@ const CycleT *getIntDivCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
return Cycle;
}
-template <typename ContextT, typename CycleT, typename BlockT,
- typename DominatorTreeT>
+template <typename ContextT, typename CycleInfoT, typename CycleT,
+ typename BlockT, typename DominatorTreeT>
const CycleT *
-getOutermostDivergentCycle(const CycleT *Cycle, const BlockT *DivTermBlock,
- const BlockT *JoinBlock, const DominatorTreeT &DT,
- ContextT &Context) {
+getOutermostDivergentCycle(const CycleInfoT &CI, const CycleT *Cycle,
+ const BlockT *DivTermBlock, const BlockT *JoinBlock,
+ const DominatorTreeT &DT, ContextT &Context) {
if (!Cycle)
return nullptr;
// First try to expand Cycle to the largest that contains JoinBlock
// but not DivTermBlock.
- const auto *Ext = getExtDivCycle(Cycle, DivTermBlock, JoinBlock);
+ const auto *Ext = getExtDivCycle(CI, Cycle, DivTermBlock, JoinBlock);
// Continue expanding to the largest cycle that contains both.
- const auto *Int = getIntDivCycle(Cycle, DivTermBlock, JoinBlock, DT, Context);
+ const auto *Int =
+ getIntDivCycle(CI, Cycle, DivTermBlock, JoinBlock, DT, Context);
if (Int)
return Int;
@@ -1079,7 +1084,7 @@ bool GenericUniformityAnalysisImpl<ContextT>::isTemporalDivergent(
const BlockT &ObservingBlock, const InstructionT &Def) const {
const BlockT *DefBlock = Def.getParent();
for (const CycleT *Cycle = CI.getCycle(DefBlock);
- Cycle && !Cycle->contains(&ObservingBlock);
+ Cycle && !CI.contains(Cycle, &ObservingBlock);
Cycle = Cycle->getParentCycle()) {
if (DivergentExitCycles.contains(Cycle)) {
return true;
@@ -1109,7 +1114,7 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
LLVM_DEBUG(dbgs() << "visiting join block " << Context.print(JoinBlock)
<< "\n");
if (const auto *Outermost = getOutermostDivergentCycle(
- Cycle, DivTermBlock, JoinBlock, DT, Context)) {
+ CI, Cycle, DivTermBlock, JoinBlock, DT, Context)) {
LLVM_DEBUG(dbgs() << "found divergent cycle\n");
DivCycles.push_back(Outermost);
continue;
@@ -1132,7 +1137,7 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
if (!insertIfNotContained(AssumedDivergent, C))
continue;
LLVM_DEBUG(dbgs() << "process divergent cycle\n");
- for (const BlockT *BB : C->blocks()) {
+ for (const BlockT *BB : CI.getBlocks(*C)) {
taintAndPushAllDefs(*BB);
}
}
@@ -1209,7 +1214,7 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
FoundDivergence = true;
OS << "CYCLES ASSUMED DIVERGENT:\n";
for (const CycleT *Cycle : AssumedDivergent) {
- OS << " " << Cycle->print(Context) << '\n';
+ OS << " " << CI.print(Cycle) << '\n';
}
}
@@ -1217,7 +1222,7 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
FoundDivergence = true;
OS << "CYCLES WITH DIVERGENT EXIT:\n";
for (const CycleT *Cycle : DivergentExitCycles) {
- OS << " " << Cycle->print(Context) << '\n';
+ OS << " " << CI.print(Cycle) << '\n';
}
}
@@ -1228,7 +1233,7 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
for (auto [Val, UseInst, Cycle] : TemporalDivergenceList) {
OS << "Value :" << Context.print(Val) << NewLine
<< "Used by :" << Context.print(UseInst) << NewLine
- << "Outside cycle :" << Cycle->print(Context) << "\n\n";
+ << "Outside cycle :" << CI.print(Cycle) << "\n\n";
}
}
@@ -1283,6 +1288,12 @@ GenericUniformityInfo<ContextT>::getFunction() const {
return DA->getFunction();
}
+template <typename ContextT>
+const typename GenericUniformityInfo<ContextT>::CycleInfoT &
+GenericUniformityInfo<ContextT>::getCycleInfo() const {
+ return DA->getCycleInfo();
+}
+
/// Whether \p V is divergent at its definition.
/// A default-constructed instance (no analysis computed) reports everything
/// as uniform, which is conservatively correct for non-divergent targets.
@@ -1338,12 +1349,12 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
NestedCycle = NestedCycle->getParentCycle();
SmallVector<BlockT *, 3> NestedExits;
- NestedCycle->getExitBlocks(NestedExits);
+ CI.getExitBlocks(*NestedCycle, NestedExits);
bool PushedNodes = false;
for (auto *NestedExitBB : NestedExits) {
LLVM_DEBUG(dbgs() << " examine exit: "
<< CI.getSSAContext().print(NestedExitBB) << "\n");
- if (Cycle && !Cycle->contains(NestedExitBB))
+ if (Cycle && !CI.contains(Cycle, NestedExitBB))
continue;
if (Finalized.count(NestedExitBB))
continue;
@@ -1366,7 +1377,7 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
for (auto *SuccBB : successors(NextBB)) {
LLVM_DEBUG(dbgs() << " examine succ: "
<< CI.getSSAContext().print(SuccBB) << "\n");
- if (Cycle && !Cycle->contains(SuccBB))
+ if (Cycle && !CI.contains(Cycle, SuccBB))
continue;
if (Finalized.count(SuccBB))
continue;
@@ -1409,7 +1420,7 @@ void ModifiedPostOrder<ContextT>::computeCyclePO(
for (auto *BB : successors(CycleHeader)) {
LLVM_DEBUG(dbgs() << " examine succ: " << CI.getSSAContext().print(BB)
<< "\n");
- if (!Cycle->contains(BB))
+ if (!CI.contains(Cycle, BB))
continue;
if (BB == CycleHeader)
continue;
diff --git a/llvm/include/llvm/ADT/GenericUniformityInfo.h b/llvm/include/llvm/ADT/GenericUniformityInfo.h
index 5ba4faa01e27e..e8d0981778165 100644
--- a/llvm/include/llvm/ADT/GenericUniformityInfo.h
+++ b/llvm/include/llvm/ADT/GenericUniformityInfo.h
@@ -57,6 +57,9 @@ template <typename ContextT> class GenericUniformityInfo {
/// The GPU kernel this analysis result is for
const FunctionT &getFunction() const;
+ /// The cycle info this analysis was computed with.
+ const CycleInfoT &getCycleInfo() const;
+
/// Whether \p V is divergent at its definition.
bool isDivergentAtDef(ConstValueRefT V) const;
diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
index 9573f922bd1b2..a9890b2fe00c0 100644
--- a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
+++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
@@ -44,9 +44,10 @@ class LLVM_ABI MachineCycleInfoWrapperPass : public MachineFunctionPass {
void print(raw_ostream &OS, const Module *M = nullptr) const override;
};
-// TODO: add this function to GenericCycle template after implementing IR
-// version.
-LLVM_ABI bool isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I);
+// TODO: add this function to the GenericCycleInfo template after implementing
+// the IR version.
+LLVM_ABI bool isCycleInvariant(const MachineCycleInfo &CI,
+ const MachineCycle &Cycle, MachineInstr &I);
class MachineCycleAnalysis : public AnalysisInfoMixin<MachineCycleAnalysis> {
friend AnalysisInfoMixin<MachineCycleAnalysis>;
diff --git a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
index ddcce17b77c8b..b5ffdda35d6cc 100644
--- a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
+++ b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
@@ -158,7 +158,7 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
return;
auto *DefBB = Token->getParent();
- if (DefBB == BB || BBCycle->contains(DefBB)) {
+ if (DefBB == BB || CI.contains(BBCycle, DefBB)) {
// degenerate occurrence of a loop intrinsic
return;
}
@@ -171,7 +171,7 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
while (true) {
auto *Parent = BBCycle->getParentCycle();
- if (!Parent || Parent->contains(DefBB))
+ if (!Parent || CI.contains(Parent, DefBB))
break;
BBCycle = Parent;
};
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index 73b8114b4fd21..f4a57df755cb2 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -264,7 +264,7 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
// ignoring any other blocks inside the loop body.
OuterL->getExitBlocks(Worklist);
} else if (OuterC) {
- OuterC->getExitBlocks(Worklist);
+ CI->getExitBlocks(*OuterC, Worklist);
} else {
Worklist.append(succ_begin(BB), succ_end(BB));
}
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index 73b6476fb7b6d..037043061df10 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -113,7 +113,7 @@ bool llvm::GenericUniformityAnalysisImpl<SSAContext>::usesValueFromCycle(
assert(!isAlwaysUniform(I));
for (const Use &U : I.operands()) {
if (auto *I = dyn_cast<Instruction>(&U)) {
- if (DefCycle.contains(I->getParent()))
+ if (CI.contains(&DefCycle, I->getParent()))
return true;
}
}
@@ -126,7 +126,7 @@ void llvm::GenericUniformityAnalysisImpl<
const Cycle &DefCycle) {
for (auto *User : I.users()) {
auto *UserInstr = cast<Instruction>(User);
- if (DefCycle.contains(UserInstr->getParent()))
+ if (CI.contains(&DefCycle, UserInstr->getParent()))
continue;
markDivergent(*UserInstr);
recordTemporalDivergence(&I, UserInstr, &DefCycle);
diff --git a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
index ded32838af106..c2f0b01c07127 100644
--- a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
@@ -116,7 +116,8 @@ MachineCycleInfoPrinterPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
}
-bool llvm::isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I) {
+bool llvm::isCycleInvariant(const MachineCycleInfo &CI,
+ const MachineCycle &Cycle, MachineInstr &I) {
MachineFunction *MF = I.getParent()->getParent();
MachineRegisterInfo *MRI = &MF->getRegInfo();
const TargetSubtargetInfo &ST = MF->getSubtarget();
@@ -150,7 +151,7 @@ bool llvm::isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I) {
} else if (!MO.isDead()) {
// A def that isn't dead can't be moved.
return false;
- } else if (any_of(Cycle->getEntries(),
+ } else if (any_of(Cycle.getEntries(),
[&](const MachineBasicBlock *Block) {
return Block->isLiveIn(Reg);
})) {
@@ -167,7 +168,7 @@ bool llvm::isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I) {
// If the cycle contains the definition of an operand, then the instruction
// isn't cycle invariant.
- if (Cycle->contains(MRI->getVRegDef(Reg)->getParent()))
+ if (CI.contains(&Cycle, MRI->getVRegDef(Reg)->getParent()))
return false;
}
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index da9eaa68d22e0..941c12fc24d31 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -733,7 +733,7 @@ void MachineSinking::FindCycleSinkCandidates(
"target\n");
continue;
}
- if (!isCycleInvariant(Cycle, MI)) {
+ if (!isCycleInvariant(*CI, *Cycle, MI)) {
LLVM_DEBUG(dbgs() << "CycleSink: Instruction is not cycle invariant\n");
continue;
}
@@ -895,7 +895,7 @@ bool MachineSinking::run(MachineFunction &MF) {
HasHighPressure = false;
for (auto *Cycle : Cycles) {
- MachineBasicBlock *Preheader = Cycle->getCyclePreheader();
+ MachineBasicBlock *Preheader = CI->getCyclePreheader(*Cycle);
if (!Preheader) {
LLVM_DEBUG(dbgs() << "CycleSink: Can't find preheader\n");
continue;
@@ -1758,7 +1758,7 @@ bool MachineSinking::aggressivelySinkIntoCycle(
return false;
LLVM_DEBUG(dbgs() << "AggressiveCycleSink: Finding sink block for: " << I);
- assert(Cycle->getCyclePreheader() && "Cycle sink needs a preheader block");
+ assert(CI->getCyclePreheader(*Cycle) && "Cycle sink needs a preheader block");
SmallVector<std::pair<RegSubRegPair, MachineInstr *>> Uses;
MachineOperand &DefMO = I.getOperand(0);
@@ -1780,7 +1780,7 @@ bool MachineSinking::aggressivelySinkIntoCycle(
"can't sink.\n");
continue;
}
- if (!Cycle->contains(MI->getParent())) {
+ if (!CI->contains(Cycle, MI->getParent())) {
LLVM_DEBUG(
dbgs() << "AggressiveCycleSink: Use not in cycle, can't sink.\n");
continue;
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index 03a046ea995ed..db8194a4e3a9c 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -122,7 +122,7 @@ bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::usesValueFromCycle(
return true;
auto *Def = F.getRegInfo().getVRegDef(Reg);
- if (DefCycle.contains(Def->getParent()))
+ if (CI.contains(&DefCycle, Def->getParent()))
return true;
}
return false;
@@ -138,7 +138,7 @@ void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::
continue;
auto Reg = Op.getReg();
for (MachineInstr &UserInstr : RegInfo.use_instructions(Reg)) {
- if (DefCycle.contains(UserInstr.getParent()))
+ if (CI.contains(&DefCycle, UserInstr.getParent()))
continue;
markDivergent(UserInstr);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
index cbb4269e17260..2734d50df387b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -251,6 +251,7 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
}
}
+ const auto &CInfo = MUI->getCycleInfo();
for (auto &LRCCacheEntry : LRCCache) {
Register Reg = LRCCacheEntry.first;
auto &CycleMergedMask = LRCCacheEntry.getSecond();
@@ -264,7 +265,7 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
for (auto Entry : Cycle->getEntries()) {
for (MachineBasicBlock *Pred : Entry->predecessors()) {
- if (!Cycle->contains(Pred)) {
+ if (!CInfo.contains(Cycle, Pred)) {
B.setInsertPt(*Pred, Pred->getFirstTerminator());
auto ImplDef = B.buildInstr(AMDGPU::IMPLICIT_DEF, {BoolS1}, {});
SSAUpdater.AddAvailableValue(Pred, ImplDef.getReg(0));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 66e252a461cd3..34cf595e8d39b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -237,7 +237,7 @@ bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
// does not contain SuccToSinkTo and also has divergent exit condition.
while (FromCycle && !FromCycle->contains(ToCycle)) {
SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
- FromCycle->getExitingBlocks(ExitingBlocks);
+ CI->getExitingBlocks(*FromCycle, ExitingBlocks);
// FromCycle has divergent exit condition.
for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 99d86781cc122..0e07946707d8f 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -11437,7 +11437,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl {
// If the incoming value is not the PHI but an instruction in the same
// cycle we might have multiple versions of it flying around.
if (CyclePHI && isa<Instruction>(V) &&
- (!C || C->contains(cast<Instruction>(V)->getParent())))
+ (!C || CI->contains(C, cast<Instruction>(V)->getParent())))
return false;
Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
diff --git a/llvm/lib/Transforms/Utils/FixIrreducible.cpp b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
index 352d5a854dbaf..c3e1ca47ac641 100644
--- a/llvm/lib/Transforms/Utils/FixIrreducible.cpp
+++ b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
@@ -220,7 +220,7 @@ static void reconnectChildLoops(LoopInfo &LI, Loop *ParentLoop, Loop *NewLoop,
}
}
-static void updateLoopInfo(LoopInfo &LI, Cycle &C,
+static void updateLoopInfo(LoopInfo &LI, Cycle &C, CycleInfo &CI,
ArrayRef<BasicBlock *> GuardBlocks) {
// The parent loop is a natural loop L mapped to the cycle header H as long as
// H is not also the header of L. In the latter case, L is destroyed and we
@@ -248,7 +248,7 @@ static void updateLoopInfo(LoopInfo &LI, Cycle &C,
NewLoop->addBasicBlockToLoop(G, LI);
}
- for (auto *BB : C.blocks()) {
+ for (auto *BB : CI.getBlocks(C)) {
NewLoop->addBlockEntry(BB);
if (LI.getLoopFor(BB) == ParentLoop) {
LLVM_DEBUG(dbgs() << "moved block from parent: " << BB->getName()
@@ -287,7 +287,7 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
// Redirect internal edges incident on the header.
BasicBlock *Header = C.getHeader();
for (BasicBlock *P : predecessors(Header)) {
- if (C.contains(P))
+ if (CI.contains(&C, P))
Predecessors.insert(P);
}
@@ -331,7 +331,7 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
Predecessors.clear();
for (BasicBlock *E : C.entries()) {
for (BasicBlock *P : predecessors(E)) {
- if (!C.contains(P))
+ if (!CI.contains(&C, P))
Predecessors.insert(P);
}
}
@@ -339,16 +339,16 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
for (BasicBlock *P : Predecessors) {
if (UncondBrInst *Branch = dyn_cast<UncondBrInst>(P->getTerminator())) {
BasicBlock *Succ0 = Branch->getSuccessor();
- Succ0 = C.contains(Succ0) ? Succ0 : nullptr;
+ Succ0 = CI.contains(&C, Succ0) ? Succ0 : nullptr;
CHub.addBranch(P, Succ0);
LLVM_DEBUG(dbgs() << "Added external branch: " << printBasicBlock(P)
<< " -> " << printBasicBlock(Succ0) << '\n');
} else if (CondBrInst *Branch = dyn_cast<CondBrInst>(P->getTerminator())) {
BasicBlock *Succ0 = Branch->getSuccessor(0);
- Succ0 = C.contains(Succ0) ? Succ0 : nullptr;
+ Succ0 = CI.contains(&C, Succ0) ? Succ0 : nullptr;
BasicBlock *Succ1 = Branch->getSuccessor(1);
- Succ1 = C.contains(Succ1) ? Succ1 : nullptr;
+ Succ1 = CI.contains(&C, Succ1) ? Succ1 : nullptr;
CHub.addBranch(P, Succ0, Succ1);
LLVM_DEBUG(dbgs() << "Added external branch: " << printBasicBlock(P)
@@ -359,7 +359,7 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
SmallDenseMap<BasicBlock *, BasicBlock *> CallBrTargets;
for (unsigned I = 0; I < CallBr->getNumSuccessors(); ++I) {
BasicBlock *Succ = CallBr->getSuccessor(I);
- if (!C.contains(Succ))
+ if (!CI.contains(&C, Succ))
continue;
auto It = CallBrTargets.find(Succ);
BasicBlock *ExistingTarget =
@@ -406,7 +406,7 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
// If we are updating LoopInfo, do that now before modifying the cycle. This
// ensures that the first guard block is the header of a new natural loop.
if (LI)
- updateLoopInfo(*LI, C, GuardBlocks);
+ updateLoopInfo(*LI, C, CI, GuardBlocks);
for (auto *G : GuardBlocks) {
LLVM_DEBUG(dbgs() << "added guard block to cycle: " << G->getName()
@@ -415,9 +415,9 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
}
C.setSingleEntry(GuardBlocks[0]);
- C.verifyCycle();
+ CI.verifyCycle(C);
if (Cycle *Parent = C.getParentCycle())
- Parent->verifyCycle();
+ CI.verifyCycle(*Parent);
LLVM_DEBUG(dbgs() << "Finished one cycle:\n"; CI.print(dbgs()););
return true;
More information about the llvm-commits
mailing list