[llvm] [CycleInfo] Represent cycles by an opaque handle. NFC (PR #210117)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 00:09:18 PDT 2026
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/210117
>From 435b118bc20e1631d75fc1864d904ed0005427c6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Thu, 16 Jul 2026 02:00:10 -0700
Subject: [PATCH 1/6] [CycleInfo] Represent cycles by an opaque handle. NFC
Consumers refer to a cycle by a GenericCycle pointer into
GenericCycleInfo's storage. Introduce GenericCycleRef, a value handle
wrapping the cycle's preorder index, and route every GenericCycleInfo
query and mutation through it. GenericCycle becomes an internal
implementation detail, so its storage representation can change without
touching callers.
Aided by Claude Opus 4.8, reviewed by Fable 5
Suggested by https://github.com/llvm/llvm-project/pull/208614#pullrequestreview-4683378136
---
.../llvm/ADT/GenericConvergenceVerifier.h | 2 +-
llvm/include/llvm/ADT/GenericCycleImpl.h | 142 +++++-----
llvm/include/llvm/ADT/GenericCycleInfo.h | 247 +++++++++++------
llvm/include/llvm/ADT/GenericUniformityImpl.h | 253 +++++++++---------
llvm/include/llvm/ADT/GenericUniformityInfo.h | 4 +-
.../llvm/CodeGen/MachineCycleAnalysis.h | 2 +-
llvm/include/llvm/IR/CycleInfo.h | 2 +-
.../llvm/IR/GenericConvergenceVerifierImpl.h | 12 +-
llvm/lib/Analysis/CFG.cpp | 16 +-
llvm/lib/Analysis/UniformityAnalysis.cpp | 2 +-
llvm/lib/CodeGen/MachineSink.cpp | 38 +--
.../lib/CodeGen/MachineUniformityAnalysis.cpp | 2 +-
.../AMDGPUGlobalISelDivergenceLowering.cpp | 12 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 12 +-
llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp | 12 +-
.../Transforms/IPO/AttributorAttributes.cpp | 10 +-
.../Scalar/DeadStoreElimination.cpp | 2 +-
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 19 +-
llvm/lib/Transforms/Utils/FixIrreducible.cpp | 14 +-
19 files changed, 442 insertions(+), 361 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
index dc9495ce57bfd..138f5d24be109 100644
--- a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
+++ b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
@@ -28,7 +28,7 @@ template <typename ContextT> class GenericConvergenceVerifier {
using InstructionT = typename ContextT::InstructionT;
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleT = typename CycleInfoT::CycleT;
+ using Cycle = typename CycleInfoT::Cycle;
void initialize(raw_ostream *OS,
function_ref<void(const Twine &Message)> FailureCB,
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 779a6eb813401..6e60282581442 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -36,10 +36,10 @@ namespace llvm {
template <typename ContextT>
void GenericCycleInfo<ContextT>::getExitBlocks(
- const CycleT &C, SmallVectorImpl<BlockT *> &TmpStorage) const {
+ Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const {
if (ExitBlocksCaches.empty())
ExitBlocksCaches.resize(NumCycles);
- auto &Cache = ExitBlocksCaches[getCycleIndex(C)];
+ auto &Cache = ExitBlocksCaches[C.Index];
if (!Cache.empty()) {
TmpStorage.append(Cache.begin(), Cache.end());
return;
@@ -66,7 +66,7 @@ void GenericCycleInfo<ContextT>::getExitBlocks(
template <typename ContextT>
void GenericCycleInfo<ContextT>::getExitingBlocks(
- const CycleT &C, SmallVectorImpl<BlockT *> &TmpStorage) const {
+ Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const {
for (BlockT *Block : getBlocks(C)) {
for (BlockT *Succ : successors(Block)) {
if (!contains(C, Succ)) {
@@ -78,8 +78,7 @@ void GenericCycleInfo<ContextT>::getExitingBlocks(
}
template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getCyclePreheader(const CycleT &C) const
- -> BlockT * {
+auto GenericCycleInfo<ContextT>::getCyclePreheader(Cycle C) const -> BlockT * {
BlockT *Predecessor = getCyclePredecessor(C);
if (!Predecessor)
return nullptr;
@@ -97,7 +96,7 @@ auto GenericCycleInfo<ContextT>::getCyclePreheader(const CycleT &C) const
}
template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getCyclePredecessor(const CycleT &C) const
+auto GenericCycleInfo<ContextT>::getCyclePredecessor(Cycle C) const
-> BlockT * {
if (!isReducible(C))
return nullptr;
@@ -118,14 +117,14 @@ auto GenericCycleInfo<ContextT>::getCyclePredecessor(const CycleT &C) const
}
template <typename ContextT>
-void GenericCycleInfo<ContextT>::verifyCycle(const CycleT &C) const {
+void GenericCycleInfo<ContextT>::verifyCycle(Cycle C) const {
#ifndef NDEBUG
assert(getNumBlocks(C) != 0 && "Cycle cannot be empty.");
DenseSet<BlockT *> Blocks;
for (BlockT *BB : getBlocks(C)) {
assert(Blocks.insert(BB).second); // duplicates in block list?
}
- assert(!C.Entries.empty() && "Cycle must have one or more entries.");
+ assert(!getEntries(C).empty() && "Cycle must have one or more entries.");
DenseSet<BlockT *> Entries;
for (BlockT *Entry : getEntries(C)) {
@@ -192,21 +191,22 @@ void GenericCycleInfo<ContextT>::verifyCycle(const CycleT &C) const {
}
template <typename ContextT>
-void GenericCycleInfo<ContextT>::verifyCycleNest(const CycleT &C) const {
+void GenericCycleInfo<ContextT>::verifyCycleNest(Cycle C) const {
#ifndef NDEBUG
+ const CycleT &Cyc = deref(C);
// Check the subcycles.
- for (CycleT *Child : children(C)) {
+ for (Cycle Child : children(C)) {
// Each block in each subcycle should be contained within this cycle.
- for (BlockT *BB : getBlocks(*Child)) {
+ for (BlockT *BB : getBlocks(Child)) {
assert(contains(C, BB) &&
"Cycle does not contain all the blocks of a subcycle!");
}
- assert(Child->Depth == C.Depth + 1);
+ assert(deref(Child).Depth == Cyc.Depth + 1);
}
// Check the parent cycle pointer.
- if (C.ParentCycle) {
- assert(is_contained(children(*C.ParentCycle), &C) &&
+ if (Cyc.ParentCycle) {
+ assert(is_contained(children(ref(*Cyc.ParentCycle)), C) &&
"Cycle is not a subcycle of its parent!");
}
#endif
@@ -297,29 +297,32 @@ void GenericCycleInfo<ContextT>::addToBlockMap(BlockT *Block, CycleT *Cycle) {
}
template <typename ContextT>
-void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleT *Cycle) {
+void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, Cycle C) {
+ CycleT &Cyc = deref(C);
// Make sure BlockMap is large enough for the new block.
unsigned Number = GraphTraits<BlockT *>::getNumber(Block);
if (Number >= BlockMap.size())
BlockMap.resize(GraphTraits<FunctionT *>::getMaxNumber(Block->getParent()));
- // Insert Block at the end of Cycle's slice and shift every later cycle's
- // range right. Ranges straddling Pos belong to Cycle's ancestors and are
+ // Insert Block at the end of Cyc's slice and shift every later cycle's
+ // range right. Ranges straddling Pos belong to Cyc's ancestors and are
// extended below.
- unsigned Pos = Cycle->IdxEnd;
+ unsigned Pos = Cyc.IdxEnd;
BlockLayout.insert(BlockLayout.begin() + Pos, Block);
- for (CycleT &C : cycles())
- if (C.IdxBegin >= Pos) {
- ++C.IdxBegin;
- ++C.IdxEnd;
+ for (unsigned I = 0; I != NumCycles; ++I) {
+ CycleT &X = Cycles[I];
+ if (X.IdxBegin >= Pos) {
+ ++X.IdxBegin;
+ ++X.IdxEnd;
}
- addToBlockMap(Block, Cycle);
- // Cycle and its ancestors gain the new block: extend each one's slice and
+ }
+ addToBlockMap(Block, &Cyc);
+ // Cyc and its ancestors gain the new block: extend each one's slice and
// invalidate its exit-block cache in a single walk up the tree.
- for (CycleT *C = Cycle; C; C = getParentCycle(*C)) {
- ++C->IdxEnd;
+ for (CycleT *P = &Cyc; P; P = P->ParentCycle) {
+ ++P->IdxEnd;
if (!ExitBlocksCaches.empty())
- ExitBlocksCaches[getCycleIndex(*C)].clear();
+ ExitBlocksCaches[getCycleIndex(*P)].clear();
}
}
@@ -442,7 +445,7 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
}
}
if (IsEntry) {
- assert(!Info.isEntry(*NewCycle, Block));
+ assert(!is_contained(NewCycle->Entries, Block));
LLVM_DEBUG(errs() << "append as entry\n");
NewCycle->appendEntry(Block);
} else {
@@ -457,23 +460,27 @@ void GenericCycleInfoCompute<ContextT>::run(FunctionT *F) {
// If the block has already been discovered by some cycle
// (possibly by ourself), then the outermost cycle containing it
- // should become our child.
- if (auto *BlockParent = Info.getTopLevelParentCycle(Block)) {
+ // should become our child. Walk the temporary forest directly:
+ // handles are not meaningful until flatten() builds the flat array.
+ CycleT *BlockParent = Info.getCyclePtr(Block);
+ while (BlockParent && BlockParent->ParentCycle)
+ BlockParent = BlockParent->ParentCycle;
+ if (BlockParent) {
LLVM_DEBUG(errs() << " block " << Info.Context.print(Block) << ": ");
if (BlockParent != NewCycle) {
- LLVM_DEBUG(errs() << "discovered child cycle "
- << Info.Context.print(Info.getHeader(*BlockParent))
- << "\n");
+ LLVM_DEBUG(errs()
+ << "discovered child cycle "
+ << Info.Context.print(BlockParent->Entries[0]) << "\n");
// Make BlockParent the child of NewCycle.
moveTopLevelCycleToNewParent(NewCycle, BlockParent);
- for (auto *ChildEntry : Info.getEntries(*BlockParent))
+ for (auto *ChildEntry : BlockParent->Entries)
ProcessPredecessors(ChildEntry);
} else {
- LLVM_DEBUG(errs() << "known child cycle "
- << Info.Context.print(Info.getHeader(*BlockParent))
- << "\n");
+ LLVM_DEBUG(errs()
+ << "known child cycle "
+ << Info.Context.print(BlockParent->Entries[0]) << "\n");
}
} else {
Info.addToBlockMap(Block, NewCycle);
@@ -580,11 +587,11 @@ void GenericCycleInfo<ContextT>::splitCriticalEdge(BlockT *Pred, BlockT *Succ,
BlockT *NewBlock) {
// Edge Pred-Succ is replaced by edges Pred-NewBlock and NewBlock-Succ, all
// cycles that had blocks Pred and Succ also get NewBlock.
- CycleT *Cycle = getSmallestCommonCycle(getCycle(Pred), getCycle(Succ));
- if (!Cycle)
+ Cycle C = getSmallestCommonCycle(getCycle(Pred), getCycle(Succ));
+ if (!C)
return;
- addBlockToCycle(NewBlock, Cycle);
+ addBlockToCycle(NewBlock, C);
verifyCycleNest();
}
@@ -593,25 +600,24 @@ void GenericCycleInfo<ContextT>::splitCriticalEdge(BlockT *Pred, BlockT *Succ,
/// \returns the innermost cycle containing both \p A and \p B
/// or nullptr if there is no such cycle.
template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(CycleT *A,
- CycleT *B) const
- -> CycleT * {
+auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(Cycle A, Cycle B) const
+ -> Cycle {
if (!A || !B)
- return nullptr;
+ return Cycle();
// If cycles A and B have different depth replace them with parent cycle
// until they have the same depth.
- while (getDepth(*A) > getDepth(*B))
- A = getParentCycle(*A);
- while (getDepth(*B) > getDepth(*A))
- B = getParentCycle(*B);
+ while (getDepth(A) > getDepth(B))
+ A = getParentCycle(A);
+ while (getDepth(B) > getDepth(A))
+ B = getParentCycle(B);
// Cycles A and B are at same depth but may be disjoint, replace them with
// parent cycles until we find cycle that contains both or we run out of
// parent cycles.
while (A != B) {
- A = getParentCycle(*A);
- B = getParentCycle(*B);
+ A = getParentCycle(A);
+ B = getParentCycle(B);
}
return A;
@@ -624,7 +630,7 @@ auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(CycleT *A,
template <typename ContextT>
auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(BlockT *A,
BlockT *B) const
- -> CycleT * {
+ -> Cycle {
return getSmallestCommonCycle(getCycle(A), getCycle(B));
}
@@ -637,18 +643,18 @@ void GenericCycleInfo<ContextT>::verifyCycleNest(bool VerifyFull) const {
#ifndef NDEBUG
DenseSet<BlockT *> CycleHeaders;
- for (const CycleT &Cycle : cycles()) {
- BlockT *Header = getHeader(Cycle);
+ for (Cycle C : cycles()) {
+ BlockT *Header = getHeader(C);
assert(CycleHeaders.insert(Header).second);
if (VerifyFull)
- verifyCycle(Cycle);
+ verifyCycle(C);
else
- verifyCycleNest(Cycle);
+ verifyCycleNest(C);
// Check the block map entries for blocks contained in this cycle.
- for (BlockT *BB : getBlocks(Cycle)) {
- CycleT *CycleInBlockMap = getCycle(BB);
- assert(CycleInBlockMap != nullptr);
- assert(contains(Cycle, *CycleInBlockMap));
+ for (BlockT *BB : getBlocks(C)) {
+ Cycle InBlockMap = getCycle(BB);
+ assert(InBlockMap.isValid());
+ assert(contains(C, InBlockMap));
}
}
#endif
@@ -662,23 +668,23 @@ template <typename ContextT> void GenericCycleInfo<ContextT>::verify() const {
/// \brief Print the cycle info.
template <typename ContextT>
void GenericCycleInfo<ContextT>::print(raw_ostream &Out) const {
- for (const CycleT &Cycle : cycles()) {
- for (unsigned I = 0; I < Cycle.Depth; ++I)
+ for (Cycle C : cycles()) {
+ for (unsigned I = 0, Depth = getDepth(C); I < Depth; ++I)
Out << " ";
- Out << print(&Cycle) << '\n';
+ Out << print(C) << '\n';
}
}
/// \brief Print a single cycle: its depth, entries, and remaining blocks.
template <typename ContextT>
-Printable GenericCycleInfo<ContextT>::print(const CycleT *Cycle) const {
- return Printable([this, Cycle](raw_ostream &Out) {
- Out << "depth=" << Cycle->Depth << ": entries("
- << printEntries(*Cycle, Context) << ')';
+Printable GenericCycleInfo<ContextT>::print(Cycle C) const {
+ return Printable([this, C](raw_ostream &Out) {
+ Out << "depth=" << getDepth(C) << ": entries(" << printEntries(C, Context)
+ << ')';
- for (auto *Block : getBlocks(*Cycle)) {
- if (isEntry(*Cycle, Block))
+ for (auto *Block : getBlocks(C)) {
+ if (isEntry(C, Block))
continue;
Out << ' ' << Context.print(Block);
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 8b706f5986307..7fc8b5706cec4 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -91,37 +91,50 @@ template <typename ContextT> class GenericCycle {
public:
GenericCycle() = default;
+};
- /// Iteration over child cycles: the first child (if any) immediately
- /// follows this cycle in the preorder array, and each next sibling follows
- /// the previous child's subtree.
- //@{
- struct const_child_iterator
- : iterator_facade_base<const_child_iterator, std::forward_iterator_tag,
- GenericCycle *, std::ptrdiff_t, GenericCycle *,
- GenericCycle *> {
- const GenericCycle *C = nullptr;
+/// Opaque handle to a cycle within a GenericCycleInfo. Wraps the cycle's
+/// preorder index; a default-constructed handle is invalid ("no cycle"). All
+/// queries live on GenericCycleInfo, which resolves the handle to storage.
+///
+/// Handles remain valid as long as the cycle forest is not recomputed.
+/// addBlockToCycle() adds a block but never adds, removes, or reorders cycles,
+/// so it leaves every handle valid.
+template <typename ContextT> class GenericCycleRef {
+ static constexpr unsigned InvalidIndex = ~0u;
+ unsigned Index = InvalidIndex;
- const_child_iterator() = default;
- explicit const_child_iterator(const GenericCycle *C) : C(C) {}
+ explicit GenericCycleRef(unsigned Index) : Index(Index) {}
+ friend class GenericCycleInfo<ContextT>;
+ friend struct DenseMapInfo<GenericCycleRef<ContextT>>;
- GenericCycle *operator*() const { return const_cast<GenericCycle *>(C); }
- const_child_iterator &operator++() {
- C += 1 + C->NumDescendants;
- return *this;
- }
- bool operator==(const const_child_iterator &Other) const {
- return C == Other.C;
- }
- };
- //@}
+public:
+ GenericCycleRef() = default;
+ bool isValid() const { return Index != InvalidIndex; }
+ explicit operator bool() const { return isValid(); }
+ bool operator==(GenericCycleRef O) const { return Index == O.Index; }
+ bool operator!=(GenericCycleRef O) const { return Index != O.Index; }
+};
+
+/// The empty/tombstone keys are distinct from the invalid handle, so an invalid
+/// handle stays a legal key -- matching the pointer world where nullptr is a
+/// legal DenseMap/SmallPtrSet key.
+template <typename ContextT> struct DenseMapInfo<GenericCycleRef<ContextT>> {
+ using T = GenericCycleRef<ContextT>;
+ static T getEmptyKey() { return T(~0u - 1); }
+ static T getTombstoneKey() { return T(~0u - 2); }
+ static unsigned getHashValue(T C) { return C.Index; }
+ static bool isEqual(T A, T B) { return A.Index == B.Index; }
};
/// \brief Cycle information for a function.
template <typename ContextT> class GenericCycleInfo {
public:
using BlockT = typename ContextT::BlockT;
+ /// The internal, by-value storage type for a cycle.
using CycleT = GenericCycle<ContextT>;
+ /// The opaque handle by which consumers refer to a cycle.
+ using Cycle = GenericCycleRef<ContextT>;
using FunctionT = typename ContextT::FunctionT;
template <typename> friend class GenericCycleInfoCompute;
@@ -149,6 +162,28 @@ template <typename ContextT> class GenericCycleInfo {
/// The preorder index of \p C, i.e. its offset in the Cycles array.
unsigned getCycleIndex(const CycleT &C) const { return &C - Cycles.get(); }
+ /// Resolve a handle to its stored cycle. The assert catches deref of an
+ /// invalid handle and (partially) of a handle from another CycleInfo.
+ CycleT &deref(Cycle C) {
+ assert(C.Index < NumCycles);
+ return Cycles[C.Index];
+ }
+ const CycleT &deref(Cycle C) const {
+ assert(C.Index < NumCycles);
+ return Cycles[C.Index];
+ }
+ /// The handle for a stored cycle.
+ Cycle ref(const CycleT &C) const { return Cycle(getCycleIndex(C)); }
+
+ /// The innermost cycle containing \p Block as a raw pointer, or null. Used
+ /// internally and during construction, where handles are not yet meaningful
+ /// because the flat array does not exist.
+ CycleT *getCyclePtr(const BlockT *Block) const {
+ verifyBlockNumberEpoch(Block->getParent());
+ unsigned Number = GraphTraits<const BlockT *>::getNumber(Block);
+ return Number < BlockMap.size() ? BlockMap[Number] : nullptr;
+ }
+
void verifyBlockNumberEpoch(const FunctionT *Fn) const {
assert(BlockNumberEpoch ==
GraphTraits<const FunctionT *>::getNumberEpoch(Fn) &&
@@ -157,6 +192,48 @@ template <typename ContextT> class GenericCycleInfo {
void addToBlockMap(BlockT *Block, CycleT *Cycle);
public:
+ /// Iteration over child cycles, yielding handles. The first child (if any)
+ /// immediately follows this cycle in the preorder array, and each next
+ /// sibling follows the previous child's subtree.
+ struct const_child_iterator
+ : iterator_facade_base<const_child_iterator, std::forward_iterator_tag,
+ Cycle, std::ptrdiff_t, Cycle, Cycle> {
+ const GenericCycleInfo *CI = nullptr;
+ unsigned Index = 0;
+
+ const_child_iterator() = default;
+ const_child_iterator(const GenericCycleInfo &CI, unsigned Index)
+ : CI(&CI), Index(Index) {}
+
+ Cycle operator*() const { return Cycle(Index); }
+ const_child_iterator &operator++() {
+ Index += 1 + CI->Cycles[Index].NumDescendants;
+ return *this;
+ }
+ bool operator==(const const_child_iterator &Other) const {
+ return Index == Other.Index;
+ }
+ };
+
+ /// Sequential iteration over all cycles in forest preorder, yielding handles.
+ struct const_cycle_iterator
+ : iterator_facade_base<const_cycle_iterator, std::forward_iterator_tag,
+ Cycle, std::ptrdiff_t, Cycle, Cycle> {
+ unsigned Index = 0;
+
+ const_cycle_iterator() = default;
+ explicit const_cycle_iterator(unsigned Index) : Index(Index) {}
+
+ Cycle operator*() const { return Cycle(Index); }
+ const_cycle_iterator &operator++() {
+ ++Index;
+ return *this;
+ }
+ bool operator==(const const_cycle_iterator &Other) const {
+ return Index == Other.Index;
+ }
+ };
+
GenericCycleInfo() = default;
GenericCycleInfo(GenericCycleInfo &&) = default;
GenericCycleInfo &operator=(GenericCycleInfo &&) = default;
@@ -169,112 +246,122 @@ template <typename ContextT> class GenericCycleInfo {
const ContextT &getSSAContext() const { return Context; }
/// All cycles in forest preorder.
- MutableArrayRef<CycleT> cycles() { return {Cycles.get(), NumCycles}; }
- ArrayRef<CycleT> cycles() const { return {Cycles.get(), NumCycles}; }
+ iterator_range<const_cycle_iterator> cycles() const {
+ return llvm::make_range(const_cycle_iterator(0),
+ const_cycle_iterator(NumCycles));
+ }
/// \brief Find the innermost cycle containing \p Block.
///
- /// \returns the innermost cycle containing \p Block or nullptr if
+ /// \returns the innermost cycle containing \p Block or an invalid handle if
/// it is not contained in any cycle.
- CycleT *getCycle(const BlockT *Block) const {
- verifyBlockNumberEpoch(Block->getParent());
- unsigned Number = GraphTraits<const BlockT *>::getNumber(Block);
- return Number < BlockMap.size() ? BlockMap[Number] : nullptr;
+ Cycle getCycle(const BlockT *Block) const {
+ CycleT *C = getCyclePtr(Block);
+ return C ? ref(*C) : Cycle();
}
- BlockT *getHeader(const CycleT &C) const { return C.Entries[0]; }
- bool isReducible(const CycleT &C) const { return C.Entries.size() == 1; }
- CycleT *getParentCycle(const CycleT &C) const { return C.ParentCycle; }
- unsigned getDepth(const CycleT &C) const { return C.Depth; }
- size_t getNumBlocks(const CycleT &C) const { return C.IdxEnd - C.IdxBegin; }
+ BlockT *getHeader(Cycle C) const { return deref(C).Entries[0]; }
+ bool isReducible(Cycle C) const { return deref(C).Entries.size() == 1; }
+ Cycle getParentCycle(Cycle C) const {
+ CycleT *P = deref(C).ParentCycle;
+ return P ? ref(*P) : Cycle();
+ }
+ unsigned getDepth(Cycle C) const { return deref(C).Depth; }
+ size_t getNumBlocks(Cycle C) const {
+ const CycleT &Cyc = deref(C);
+ return Cyc.IdxEnd - Cyc.IdxBegin;
+ }
- ArrayRef<BlockT *> getEntries(const CycleT &C) const { return C.Entries; }
- bool isEntry(const CycleT &C, const BlockT *Block) const {
- return is_contained(C.Entries, Block);
+ ArrayRef<BlockT *> getEntries(Cycle C) const { return deref(C).Entries; }
+ bool isEntry(Cycle C, const BlockT *Block) const {
+ return is_contained(deref(C).Entries, Block);
}
- void setSingleEntry(CycleT &C, BlockT *Block) {
- C.Entries.clear();
- C.Entries.push_back(Block);
+ void setSingleEntry(Cycle C, BlockT *Block) {
+ auto &Entries = deref(C).Entries;
+ Entries.clear();
+ Entries.push_back(Block);
}
/// Returns true iff \p Outer contains \p Inner. O(1). Non-strict.
- bool contains(const CycleT &Outer, const CycleT &Inner) const {
- return Outer.IdxBegin <= Inner.IdxBegin && Inner.IdxEnd <= Outer.IdxEnd;
+ bool contains(Cycle Outer, Cycle Inner) const {
+ const CycleT &O = deref(Outer);
+ const CycleT &I = deref(Inner);
+ return O.IdxBegin <= I.IdxBegin && I.IdxEnd <= O.IdxEnd;
}
- iterator_range<typename CycleT::const_child_iterator>
- children(const CycleT &C) const {
+ iterator_range<const_child_iterator> children(Cycle C) const {
+ unsigned First = C.Index + 1;
return llvm::make_range(
- typename CycleT::const_child_iterator{&C + 1},
- typename CycleT::const_child_iterator{&C + 1 + C.NumDescendants});
+ const_child_iterator(*this, First),
+ const_child_iterator(*this, First + deref(C).NumDescendants));
}
- Printable printEntries(const CycleT &C, const ContextT &Ctx) const {
- return Printable([&C, &Ctx](raw_ostream &Out) {
+ Printable printEntries(Cycle C, const ContextT &Ctx) const {
+ return Printable([this, C, &Ctx](raw_ostream &Out) {
ListSeparator LS(" ");
- for (auto *Entry : C.Entries)
+ for (auto *Entry : deref(C).Entries)
Out << LS << Ctx.print(Entry);
});
}
/// \brief Return whether \p Block is contained in \p C. O(1).
- bool contains(const CycleT &C, const BlockT *Block) const {
- const CycleT *Inner = getCycle(Block);
- return Inner && contains(C, *Inner);
+ bool contains(Cycle C, const BlockT *Block) const {
+ Cycle Inner = getCycle(Block);
+ return Inner.isValid() && contains(C, Inner);
}
/// \brief Return the blocks of \p C, including those of nested cycles.
- ArrayRef<BlockT *> getBlocks(const CycleT &C) const {
- return ArrayRef<BlockT *>(BlockLayout.begin() + C.IdxBegin,
- BlockLayout.begin() + C.IdxEnd);
+ ArrayRef<BlockT *> getBlocks(Cycle C) const {
+ const CycleT &Cyc = deref(C);
+ return ArrayRef<BlockT *>(BlockLayout.begin() + Cyc.IdxBegin,
+ BlockLayout.begin() + Cyc.IdxEnd);
}
- CycleT *getSmallestCommonCycle(CycleT *A, CycleT *B) const;
- CycleT *getSmallestCommonCycle(BlockT *A, BlockT *B) const;
+ Cycle getSmallestCommonCycle(Cycle A, Cycle B) const;
+ Cycle getSmallestCommonCycle(BlockT *A, BlockT *B) const;
/// \brief Return the depth of the innermost cycle containing \p Block, or 0
/// if it is not contained in any cycle.
unsigned getCycleDepth(const BlockT *Block) const {
- CycleT *Cycle = getCycle(Block);
- return Cycle ? getDepth(*Cycle) : 0;
+ Cycle C = getCycle(Block);
+ return C.isValid() ? getDepth(C) : 0;
}
- CycleT *getTopLevelParentCycle(const BlockT *Block) const {
- CycleT *Cycle = getCycle(Block);
- while (Cycle && Cycle->ParentCycle)
- Cycle = Cycle->ParentCycle;
- return Cycle;
+ Cycle getTopLevelParentCycle(const BlockT *Block) const {
+ CycleT *C = getCyclePtr(Block);
+ if (!C)
+ return Cycle();
+ while (C->ParentCycle)
+ C = C->ParentCycle;
+ return ref(*C);
}
/// Return all of the successor blocks of \p C: the blocks outside of \p C
/// which are branched to from within it.
- void getExitBlocks(const CycleT &C,
- SmallVectorImpl<BlockT *> &TmpStorage) const;
+ void getExitBlocks(Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const;
/// Return all blocks of \p C that have a successor outside of \p C.
- void getExitingBlocks(const CycleT &C,
- SmallVectorImpl<BlockT *> &TmpStorage) const;
+ void getExitingBlocks(Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const;
/// Return the preheader block for \p C. Pre-header is well-defined for
/// reducible cycle in docs/LoopTerminology.md as: the only one entering
/// block and its only edge is to the entry block. Return null for
/// irreducible cycles.
- BlockT *getCyclePreheader(const CycleT &C) const;
+ BlockT *getCyclePreheader(Cycle C) const;
/// If \p C has exactly one entry with exactly one predecessor, return it,
/// otherwise return nullptr.
- BlockT *getCyclePredecessor(const CycleT &C) const;
+ BlockT *getCyclePredecessor(Cycle C) const;
/// Verify that \p C is actually a well-formed cycle in the CFG.
- void verifyCycle(const CycleT &C) const;
+ void verifyCycle(Cycle C) const;
/// Verify the parent-child relations of \p C.
///
/// Note that this does \em not check that \p C is really a cycle in the CFG.
- void verifyCycleNest(const CycleT &C) const;
+ void verifyCycleNest(Cycle C) const;
- /// Assumes that \p Cycle is the innermost cycle containing \p Block.
- /// \p Block will be appended to \p Cycle and all of its parent cycles.
- /// \p Block will be added to BlockMap with \p Cycle and
- /// BlockMapTopLevel with \p Cycle's top level parent cycle.
- void addBlockToCycle(BlockT *Block, CycleT *Cycle);
+ /// Assumes that \p C is the innermost cycle containing \p Block.
+ /// \p Block will be appended to \p C and all of its parent cycles.
+ /// \p Block will be added to BlockMap with \p C.
+ void addBlockToCycle(BlockT *Block, Cycle C);
/// Methods for debug and self-test.
//@{
@@ -282,18 +369,18 @@ template <typename ContextT> class GenericCycleInfo {
void verify() const;
void print(raw_ostream &Out) const;
void dump() const { print(dbgs()); }
- Printable print(const CycleT *Cycle) const;
+ Printable print(Cycle C) const;
//@}
/// Iteration over top-level cycles.
//@{
- using const_toplevel_iterator = typename CycleT::const_child_iterator;
+ using const_toplevel_iterator = const_child_iterator;
const_toplevel_iterator toplevel_begin() const {
- return const_toplevel_iterator{Cycles.get()};
+ return const_toplevel_iterator(*this, 0);
}
const_toplevel_iterator toplevel_end() const {
- return const_toplevel_iterator{Cycles.get() + NumCycles};
+ return const_toplevel_iterator(*this, NumCycles);
}
iterator_range<const_toplevel_iterator> toplevel_cycles() const {
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 3f78b21edd8a9..34f53084fdf7f 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -94,7 +94,7 @@ template <typename ContextT> class ModifiedPostOrder {
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleT = typename CycleInfoT::CycleT;
+ using Cycle = typename CycleInfoT::Cycle;
using const_iterator = typename std::vector<BlockT *>::const_iterator;
ModifiedPostOrder(const ContextT &C) : Context(C) {}
@@ -132,11 +132,11 @@ template <typename ContextT> class ModifiedPostOrder {
SmallPtrSet<const BlockT *, 32> ReducibleCycleHeaders;
const ContextT &Context;
- void computeCyclePO(const CycleInfoT &CI, const CycleT *Cycle,
+ void computeCyclePO(const CycleInfoT &CI, Cycle C,
SmallPtrSetImpl<const BlockT *> &Finalized);
void computeStackPO(SmallVectorImpl<const BlockT *> &Stack,
- const CycleInfoT &CI, const CycleT *Cycle,
+ const CycleInfoT &CI, Cycle C,
SmallPtrSetImpl<const BlockT *> &Finalized);
};
@@ -269,7 +269,7 @@ template <typename ContextT> class GenericSyncDependenceAnalysis {
using InstructionT = typename ContextT::InstructionT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleT = typename CycleInfoT::CycleT;
+ using Cycle = typename CycleInfoT::Cycle;
using ConstBlockSet = SmallPtrSet<const BlockT *, 4>;
using ModifiedPO = ModifiedPostOrder<ContextT>;
@@ -339,7 +339,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleT = typename CycleInfoT::CycleT;
+ using Cycle = typename CycleInfoT::Cycle;
using SyncDependenceAnalysisT = GenericSyncDependenceAnalysis<ContextT>;
using DivergenceDescriptorT =
@@ -347,7 +347,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
using BlockLabelMapT = typename SyncDependenceAnalysisT::BlockLabelMap;
using TemporalDivergenceTuple =
- std::tuple<ConstValueRefT, InstructionT *, const CycleT *>;
+ std::tuple<ConstValueRefT, InstructionT *, Cycle>;
GenericUniformityAnalysisImpl(const DominatorTreeT &DT, const CycleInfoT &CI,
const TargetTransformInfo *TTI)
@@ -420,8 +420,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
SmallVector<TemporalDivergenceTuple, 8> TemporalDivergenceList;
- void recordTemporalDivergence(ConstValueRefT, const InstructionT *,
- const CycleT *);
+ void recordTemporalDivergence(ConstValueRefT, const InstructionT *, Cycle);
/// Check if an instruction with Custom uniformity can be proven uniform
/// based on its operands. This queries the target-specific callback.
@@ -463,13 +462,13 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
const DominatorTreeT &DT;
// Recognized cycles with divergent exits.
- SmallSetVector<const CycleT *, 8> DivergentExitCycles;
+ SmallSetVector<Cycle, 8> DivergentExitCycles;
// Cycles assumed to be divergent.
//
// We don't use a set here because every insertion needs an explicit
// traversal of all existing members.
- SmallVector<const CycleT *> AssumedDivergent;
+ SmallVector<Cycle> AssumedDivergent;
// The SDA links divergent branches to divergent control-flow joins.
SyncDependenceAnalysisT SDA;
@@ -488,21 +487,20 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
/// \brief Identify all Instructions that become divergent because \p DivExit
/// is a divergent cycle exit of \p DivCycle. Mark those instructions as
/// divergent and push them on the worklist.
- void propagateCycleExitDivergence(const BlockT &DivExit,
- const CycleT &DivCycle);
+ void propagateCycleExitDivergence(const BlockT &DivExit, Cycle DivCycle);
/// Mark as divergent all external uses of values defined in \p DefCycle.
- void analyzeCycleExitDivergence(const CycleT &DefCycle);
+ void analyzeCycleExitDivergence(Cycle DefCycle);
/// \brief Mark as divergent all uses of \p I that are outside \p DefCycle.
void propagateTemporalDivergence(const InstructionT &I,
- const CycleT &DefCycle);
+ const Cycle &DefCycle);
/// \brief Push all users of \p Val (in the region) to the worklist.
void pushUsers(const InstructionT &I);
void pushUsers(ConstValueRefT V);
- bool usesValueFromCycle(const InstructionT &I, const CycleT &DefCycle) const;
+ bool usesValueFromCycle(const InstructionT &I, const Cycle &DefCycle) const;
/// \brief Whether \p Def is divergent when read in \p ObservingBlock.
bool isTemporalDivergent(const BlockT &ObservingBlock,
@@ -523,7 +521,7 @@ template <typename ContextT> class DivergencePropagator {
using ValueRefT = typename ContextT::ValueRefT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleT = typename CycleInfoT::CycleT;
+ using Cycle = typename CycleInfoT::Cycle;
using ModifiedPO = ModifiedPostOrder<ContextT>;
using SyncDependenceAnalysisT = GenericSyncDependenceAnalysis<ContextT>;
@@ -637,30 +635,29 @@ template <typename ContextT> class DivergencePropagator {
<< Context.print(&DivTermBlock) << "\n");
int DivTermIdx = CyclePOT.getIndex(&DivTermBlock);
- auto const *DivTermCycle = CI.getCycle(&DivTermBlock);
+ Cycle DivTermCycle = CI.getCycle(&DivTermBlock);
// Locate the largest ancestor cycle that is not reducible and does not
// contain a reducible ancestor. This is done with a lambda that is defined
// and invoked in the same statement.
- const CycleT *IrreducibleAncestor =
- [this](const CycleT *C) -> const CycleT * {
+ Cycle IrreducibleAncestor = [this](Cycle C) -> Cycle {
if (!C)
- return nullptr;
- if (CI.isReducible(*C))
- return nullptr;
- while (const CycleT *P = CI.getParentCycle(*C)) {
- if (CI.isReducible(*P))
+ return Cycle();
+ if (CI.isReducible(C))
+ return Cycle();
+ while (Cycle P = CI.getParentCycle(C)) {
+ if (CI.isReducible(P))
return C;
C = P;
}
- assert(!CI.getParentCycle(*C));
- assert(!CI.isReducible(*C));
+ assert(!CI.getParentCycle(C));
+ assert(!CI.isReducible(C));
return C;
}(DivTermCycle);
// Bootstrap with branch targets
for (const auto *SuccBlock : successors(&DivTermBlock)) {
- if (DivTermCycle && !CI.contains(*DivTermCycle, SuccBlock)) {
+ if (DivTermCycle && !CI.contains(DivTermCycle, SuccBlock)) {
// If DivTerm exits the cycle immediately, computeJoin() might
// not reach SuccBlock with a different label. We need to
// check for this exit now.
@@ -685,7 +682,7 @@ template <typename ContextT> class DivergencePropagator {
// If no irreducible cycle, stop if freshLable.count() = 1 and Block
// is the IPD. If it is in any irreducible cycle, continue propagation.
if (FreshLabels.count() == 1 &&
- (!IrreducibleAncestor || !CI.contains(*IrreducibleAncestor, Block)))
+ (!IrreducibleAncestor || !CI.contains(IrreducibleAncestor, Block)))
break;
LLVM_DEBUG(dbgs() << "Current labels:\n"; printDefs(dbgs()));
@@ -719,11 +716,11 @@ template <typename ContextT> class DivergencePropagator {
LLVM_DEBUG(dbgs() << "Check for reducible cycle: " << Context.print(Block)
<< '\n');
if (CyclePOT.isReducibleCycleHeader(Block)) {
- const auto *BlockCycle = CI.getCycle(Block);
+ Cycle BlockCycle = CI.getCycle(Block);
LLVM_DEBUG(dbgs() << CI.print(BlockCycle) << '\n');
SmallVector<BlockT *, 4> BlockCycleExits;
- CI.getExitBlocks(*BlockCycle, BlockCycleExits);
- bool BranchIsInside = CI.contains(*BlockCycle, &DivTermBlock);
+ CI.getExitBlocks(BlockCycle, BlockCycleExits);
+ bool BranchIsInside = CI.contains(BlockCycle, &DivTermBlock);
for (auto *BlockCycleExit : BlockCycleExits) {
if (BranchIsInside)
visitCycleExitEdge(*BlockCycleExit, *Label);
@@ -741,16 +738,15 @@ template <typename ContextT> class DivergencePropagator {
// Check every cycle containing DivTermBlock for exit divergence.
// A cycle has exit divergence if the label of an exit block does
// not match the label of its header.
- for (const auto *Cycle = CI.getCycle(&DivTermBlock); Cycle;
- Cycle = CI.getParentCycle(*Cycle)) {
- if (CI.isReducible(*Cycle)) {
+ for (Cycle C = CI.getCycle(&DivTermBlock); C; C = CI.getParentCycle(C)) {
+ if (CI.isReducible(C)) {
// The exit divergence of a reducible cycle is recorded while
// propagating labels.
continue;
}
SmallVector<BlockT *> Exits;
- CI.getExitBlocks(*Cycle, Exits);
- auto *Header = CI.getHeader(*Cycle);
+ CI.getExitBlocks(C, Exits);
+ auto *Header = CI.getHeader(C);
auto *HeaderLabel = BlockLabels[Header];
for (const auto *Exit : Exits) {
if (BlockLabels[Exit] != HeaderLabel) {
@@ -878,7 +874,7 @@ void GenericUniformityAnalysisImpl<ContextT>::addCustomUniformityCandidate(
// need to be propagated as divergent at their use outside the cycle.
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
- const CycleT &DefCycle) {
+ Cycle DefCycle) {
SmallVector<BlockT *> Exits;
CI.getExitBlocks(DefCycle, Exits);
for (auto *Exit : Exits) {
@@ -901,36 +897,36 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::propagateCycleExitDivergence(
- const BlockT &DivExit, const CycleT &InnerDivCycle) {
+ const BlockT &DivExit, Cycle InnerDivCycle) {
LLVM_DEBUG(dbgs() << "\tpropCycleExitDiv " << Context.print(&DivExit)
<< "\n");
- auto *DivCycle = &InnerDivCycle;
- auto *OuterDivCycle = DivCycle;
- auto *ExitLevelCycle = CI.getCycle(&DivExit);
+ Cycle DivCycle = InnerDivCycle;
+ Cycle OuterDivCycle = DivCycle;
+ Cycle ExitLevelCycle = CI.getCycle(&DivExit);
const unsigned CycleExitDepth =
- ExitLevelCycle ? CI.getDepth(*ExitLevelCycle) : 0;
+ ExitLevelCycle ? CI.getDepth(ExitLevelCycle) : 0;
// Find outer-most cycle that does not contain \p DivExit
- while (DivCycle && CI.getDepth(*DivCycle) > CycleExitDepth) {
+ while (DivCycle && CI.getDepth(DivCycle) > CycleExitDepth) {
LLVM_DEBUG(dbgs() << " Found exiting cycle: "
- << Context.print(CI.getHeader(*DivCycle)) << "\n");
+ << Context.print(CI.getHeader(DivCycle)) << "\n");
OuterDivCycle = DivCycle;
- DivCycle = CI.getParentCycle(*DivCycle);
+ DivCycle = CI.getParentCycle(DivCycle);
}
LLVM_DEBUG(dbgs() << "\tOuter-most exiting cycle: "
- << Context.print(CI.getHeader(*OuterDivCycle)) << "\n");
+ << Context.print(CI.getHeader(OuterDivCycle)) << "\n");
if (!DivergentExitCycles.insert(OuterDivCycle))
return;
// Exit divergence does not matter if the cycle itself is assumed to
// be divergent.
- for (const auto *C : AssumedDivergent) {
- if (CI.contains(*C, *OuterDivCycle))
+ for (Cycle C : AssumedDivergent) {
+ if (CI.contains(C, OuterDivCycle))
return;
}
- analyzeCycleExitDivergence(*OuterDivCycle);
+ analyzeCycleExitDivergence(OuterDivCycle);
}
template <typename ContextT>
@@ -972,10 +968,9 @@ void GenericUniformityAnalysisImpl<ContextT>::taintAndPushPhiNodes(
///
/// \return true iff \p Candidate was added to \p Cycles.
template <typename CycleInfoT, typename CycleT>
-bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleT *> &Cycles,
- CycleT *Candidate) {
- if (llvm::any_of(Cycles,
- [&](CycleT *C) { return CI.contains(*C, *Candidate); }))
+bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleT> &Cycles,
+ CycleT Candidate) {
+ if (llvm::any_of(Cycles, [&](CycleT C) { return CI.contains(C, Candidate); }))
return false;
Cycles.push_back(Candidate);
return true;
@@ -987,31 +982,30 @@ bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleT *> &Cycles,
/// inside that cycle, then that whole cycle is assumed to be
/// divergent. This does not apply if the cycle is reducible.
template <typename CycleInfoT, typename CycleT, typename BlockT>
-const CycleT *getExtDivCycle(const CycleInfoT &CI, const CycleT *Cycle,
- const BlockT *DivTermBlock,
- const BlockT *JoinBlock) {
+CycleT getExtDivCycle(const CycleInfoT &CI, CycleT Cycle,
+ const BlockT *DivTermBlock, const BlockT *JoinBlock) {
assert(Cycle);
- assert(CI.contains(*Cycle, JoinBlock));
+ assert(CI.contains(Cycle, JoinBlock));
- if (CI.contains(*Cycle, DivTermBlock))
- return nullptr;
+ if (CI.contains(Cycle, DivTermBlock))
+ return CycleT();
- const auto *OriginalCycle = Cycle;
- const auto *Parent = CI.getParentCycle(*Cycle);
- while (Parent && !CI.contains(*Parent, DivTermBlock)) {
+ CycleT OriginalCycle = Cycle;
+ CycleT Parent = CI.getParentCycle(Cycle);
+ while (Parent && !CI.contains(Parent, DivTermBlock)) {
Cycle = Parent;
- Parent = CI.getParentCycle(*Cycle);
+ Parent = CI.getParentCycle(Cycle);
}
// If the original cycle is not the outermost cycle, then the outermost cycle
// is irreducible. If the outermost cycle were reducible, then external
// diverged paths would not reach the original inner cycle.
(void)OriginalCycle;
- assert(Cycle == OriginalCycle || !CI.isReducible(*Cycle));
+ assert(Cycle == OriginalCycle || !CI.isReducible(Cycle));
- if (CI.isReducible(*Cycle)) {
- assert(CI.getHeader(*Cycle) == JoinBlock);
- return nullptr;
+ if (CI.isReducible(Cycle)) {
+ assert(CI.getHeader(Cycle) == JoinBlock);
+ return CycleT();
}
LLVM_DEBUG(dbgs() << "cycle made divergent by external branch\n");
@@ -1024,36 +1018,35 @@ const CycleT *getExtDivCycle(const CycleInfoT &CI, const CycleT *Cycle,
/// docs/ConvergenceAnalysis.html.
template <typename ContextT, typename CycleInfoT, typename CycleT,
typename BlockT, typename DominatorTreeT>
-const CycleT *getIntDivCycle(const CycleInfoT &CI, const CycleT *Cycle,
- const BlockT *DivTermBlock,
- const BlockT *JoinBlock, const DominatorTreeT &DT,
- ContextT &Context) {
+CycleT getIntDivCycle(const CycleInfoT &CI, CycleT Cycle,
+ const BlockT *DivTermBlock, const BlockT *JoinBlock,
+ const DominatorTreeT &DT, ContextT &Context) {
LLVM_DEBUG(dbgs() << "examine join " << Context.print(JoinBlock)
<< " for internal branch " << Context.print(DivTermBlock)
<< "\n");
if (DT.properlyDominates(DivTermBlock, JoinBlock))
- return nullptr;
+ return CycleT();
// Find the smallest common cycle, if one exists.
- assert(Cycle && CI.contains(*Cycle, JoinBlock));
- while (Cycle && !CI.contains(*Cycle, DivTermBlock)) {
- Cycle = CI.getParentCycle(*Cycle);
+ assert(Cycle && CI.contains(Cycle, JoinBlock));
+ while (Cycle && !CI.contains(Cycle, DivTermBlock)) {
+ Cycle = CI.getParentCycle(Cycle);
}
- if (!Cycle || CI.isReducible(*Cycle))
- return nullptr;
+ if (!Cycle || CI.isReducible(Cycle))
+ return CycleT();
- if (DT.properlyDominates(CI.getHeader(*Cycle), JoinBlock))
- return nullptr;
+ if (DT.properlyDominates(CI.getHeader(Cycle), JoinBlock))
+ return CycleT();
- LLVM_DEBUG(dbgs() << " header " << Context.print(CI.getHeader(*Cycle))
+ LLVM_DEBUG(dbgs() << " header " << Context.print(CI.getHeader(Cycle))
<< " does not dominate join\n");
- const auto *Parent = CI.getParentCycle(*Cycle);
- while (Parent && !DT.properlyDominates(CI.getHeader(*Parent), JoinBlock)) {
- LLVM_DEBUG(dbgs() << " header " << Context.print(CI.getHeader(*Parent))
+ CycleT Parent = CI.getParentCycle(Cycle);
+ while (Parent && !DT.properlyDominates(CI.getHeader(Parent), JoinBlock)) {
+ LLVM_DEBUG(dbgs() << " header " << Context.print(CI.getHeader(Parent))
<< " does not dominate join\n");
Cycle = Parent;
- Parent = CI.getParentCycle(*Parent);
+ Parent = CI.getParentCycle(Parent);
}
LLVM_DEBUG(dbgs() << " cycle made divergent by internal branch\n");
@@ -1062,20 +1055,19 @@ const CycleT *getIntDivCycle(const CycleInfoT &CI, const CycleT *Cycle,
template <typename ContextT, typename CycleInfoT, typename CycleT,
typename BlockT, typename DominatorTreeT>
-const CycleT *
-getOutermostDivergentCycle(const CycleInfoT &CI, const CycleT *Cycle,
- const BlockT *DivTermBlock, const BlockT *JoinBlock,
- const DominatorTreeT &DT, ContextT &Context) {
+CycleT getOutermostDivergentCycle(const CycleInfoT &CI, CycleT Cycle,
+ const BlockT *DivTermBlock,
+ const BlockT *JoinBlock,
+ const DominatorTreeT &DT, ContextT &Context) {
if (!Cycle)
- return nullptr;
+ return CycleT();
// First try to expand Cycle to the largest that contains JoinBlock
// but not DivTermBlock.
- const auto *Ext = getExtDivCycle(CI, Cycle, DivTermBlock, JoinBlock);
+ CycleT Ext = getExtDivCycle(CI, Cycle, DivTermBlock, JoinBlock);
// Continue expanding to the largest cycle that contains both.
- const auto *Int =
- getIntDivCycle(CI, Cycle, DivTermBlock, JoinBlock, DT, Context);
+ CycleT Int = getIntDivCycle(CI, Cycle, DivTermBlock, JoinBlock, DT, Context);
if (Int)
return Int;
@@ -1086,10 +1078,9 @@ template <typename ContextT>
bool GenericUniformityAnalysisImpl<ContextT>::isTemporalDivergent(
const BlockT &ObservingBlock, const InstructionT &Def) const {
const BlockT *DefBlock = Def.getParent();
- for (const CycleT *Cycle = CI.getCycle(DefBlock);
- Cycle && !CI.contains(*Cycle, &ObservingBlock);
- Cycle = CI.getParentCycle(*Cycle)) {
- if (DivergentExitCycles.contains(Cycle)) {
+ for (Cycle C = CI.getCycle(DefBlock); C && !CI.contains(C, &ObservingBlock);
+ C = CI.getParentCycle(C)) {
+ if (DivergentExitCycles.contains(C)) {
return true;
}
}
@@ -1109,15 +1100,15 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
return;
const auto &DivDesc = SDA.getJoinBlocks(DivTermBlock);
- SmallVector<const CycleT *> DivCycles;
+ SmallVector<Cycle> DivCycles;
// Iterate over all blocks now reachable by a disjoint path join
for (const auto *JoinBlock : DivDesc.JoinDivBlocks) {
- const auto *Cycle = CI.getCycle(JoinBlock);
+ Cycle C = CI.getCycle(JoinBlock);
LLVM_DEBUG(dbgs() << "visiting join block " << Context.print(JoinBlock)
<< "\n");
- if (const auto *Outermost = getOutermostDivergentCycle(
- CI, Cycle, DivTermBlock, JoinBlock, DT, Context)) {
+ if (Cycle Outermost = getOutermostDivergentCycle(CI, C, DivTermBlock,
+ JoinBlock, DT, Context)) {
LLVM_DEBUG(dbgs() << "found divergent cycle\n");
DivCycles.push_back(Outermost);
continue;
@@ -1127,8 +1118,8 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
// Sort by order of decreasing depth. This allows later cycles to be skipped
// because they are already contained in earlier ones.
- llvm::sort(DivCycles, [this](const CycleT *A, const CycleT *B) {
- return CI.getDepth(*A) > CI.getDepth(*B);
+ llvm::sort(DivCycles, [this](Cycle A, Cycle B) {
+ return CI.getDepth(A) > CI.getDepth(B);
});
// Cycles that are assumed divergent due to the diverged entry
@@ -1136,19 +1127,19 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
// the DFS chosen. Conservatively, all values produced in such a
// cycle are assumed divergent. "Cycle invariant" values may be
// assumed uniform, but that requires further analysis.
- for (auto *C : DivCycles) {
+ for (Cycle C : DivCycles) {
if (!insertIfNotContained(CI, AssumedDivergent, C))
continue;
LLVM_DEBUG(dbgs() << "process divergent cycle\n");
- for (const BlockT *BB : CI.getBlocks(*C)) {
+ for (const BlockT *BB : CI.getBlocks(C)) {
taintAndPushAllDefs(*BB);
}
}
- const auto *BranchCycle = CI.getCycle(DivTermBlock);
+ Cycle BranchCycle = CI.getCycle(DivTermBlock);
assert(DivDesc.CycleDivBlocks.empty() || BranchCycle);
for (const auto *DivExitBlock : DivDesc.CycleDivBlocks) {
- propagateCycleExitDivergence(*DivExitBlock, *BranchCycle);
+ propagateCycleExitDivergence(*DivExitBlock, BranchCycle);
}
}
@@ -1177,9 +1168,8 @@ void GenericUniformityAnalysisImpl<ContextT>::compute() {
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::recordTemporalDivergence(
- ConstValueRefT Val, const InstructionT *User, const CycleT *Cycle) {
- TemporalDivergenceList.emplace_back(Val, const_cast<InstructionT *>(User),
- Cycle);
+ ConstValueRefT Val, const InstructionT *User, Cycle C) {
+ TemporalDivergenceList.emplace_back(Val, const_cast<InstructionT *>(User), C);
}
template <typename ContextT>
@@ -1216,16 +1206,16 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
if (!AssumedDivergent.empty()) {
FoundDivergence = true;
OS << "CYCLES ASSUMED DIVERGENT:\n";
- for (const CycleT *Cycle : AssumedDivergent) {
- OS << " " << CI.print(Cycle) << '\n';
+ for (Cycle C : AssumedDivergent) {
+ OS << " " << CI.print(C) << '\n';
}
}
if (!DivergentExitCycles.empty()) {
FoundDivergence = true;
OS << "CYCLES WITH DIVERGENT EXIT:\n";
- for (const CycleT *Cycle : DivergentExitCycles) {
- OS << " " << CI.print(Cycle) << '\n';
+ for (Cycle C : DivergentExitCycles) {
+ OS << " " << CI.print(C) << '\n';
}
}
@@ -1233,10 +1223,10 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
FoundDivergence = true;
OS << "\nTEMPORAL DIVERGENCE LIST:\n";
- for (auto [Val, UseInst, Cycle] : TemporalDivergenceList) {
+ for (auto [Val, UseInst, C] : TemporalDivergenceList) {
OS << "Value :" << Context.print(Val) << NewLine
<< "Used by :" << Context.print(UseInst) << NewLine
- << "Outside cycle :" << CI.print(Cycle) << "\n\n";
+ << "Outside cycle :" << CI.print(C) << "\n\n";
}
}
@@ -1334,8 +1324,8 @@ void GenericUniformityInfo<ContextT>::print(raw_ostream &Out) const {
template <typename ContextT>
void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
- SmallVectorImpl<const BlockT *> &Stack, const CycleInfoT &CI,
- const CycleT *Cycle, SmallPtrSetImpl<const BlockT *> &Finalized) {
+ SmallVectorImpl<const BlockT *> &Stack, const CycleInfoT &CI, Cycle C,
+ SmallPtrSetImpl<const BlockT *> &Finalized) {
LLVM_DEBUG(dbgs() << "inside computeStackPO\n");
while (!Stack.empty()) {
auto *NextBB = Stack.back();
@@ -1345,20 +1335,20 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
}
LLVM_DEBUG(dbgs() << " visiting " << CI.getSSAContext().print(NextBB)
<< "\n");
- auto *NestedCycle = CI.getCycle(NextBB);
- if (Cycle != NestedCycle &&
- (!Cycle || (NestedCycle && CI.contains(*Cycle, *NestedCycle)))) {
+ Cycle NestedCycle = CI.getCycle(NextBB);
+ if (C != NestedCycle &&
+ (!C || (NestedCycle && CI.contains(C, NestedCycle)))) {
LLVM_DEBUG(dbgs() << " found a cycle\n");
- while (CI.getParentCycle(*NestedCycle) != Cycle)
- NestedCycle = CI.getParentCycle(*NestedCycle);
+ while (CI.getParentCycle(NestedCycle) != C)
+ NestedCycle = CI.getParentCycle(NestedCycle);
SmallVector<BlockT *, 3> NestedExits;
- CI.getExitBlocks(*NestedCycle, NestedExits);
+ CI.getExitBlocks(NestedCycle, NestedExits);
bool PushedNodes = false;
for (auto *NestedExitBB : NestedExits) {
LLVM_DEBUG(dbgs() << " examine exit: "
<< CI.getSSAContext().print(NestedExitBB) << "\n");
- if (Cycle && !CI.contains(*Cycle, NestedExitBB))
+ if (C && !CI.contains(C, NestedExitBB))
continue;
if (Finalized.count(NestedExitBB))
continue;
@@ -1381,7 +1371,7 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
for (auto *SuccBB : successors(NextBB)) {
LLVM_DEBUG(dbgs() << " examine succ: "
<< CI.getSSAContext().print(SuccBB) << "\n");
- if (Cycle && !CI.contains(*Cycle, SuccBB))
+ if (C && !CI.contains(C, SuccBB))
continue;
if (Finalized.count(SuccBB))
continue;
@@ -1404,11 +1394,10 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
template <typename ContextT>
void ModifiedPostOrder<ContextT>::computeCyclePO(
- const CycleInfoT &CI, const CycleT *Cycle,
- SmallPtrSetImpl<const BlockT *> &Finalized) {
+ const CycleInfoT &CI, Cycle C, SmallPtrSetImpl<const BlockT *> &Finalized) {
LLVM_DEBUG(dbgs() << "inside computeCyclePO\n");
SmallVector<const BlockT *> Stack;
- auto *CycleHeader = CI.getHeader(*Cycle);
+ auto *CycleHeader = CI.getHeader(C);
LLVM_DEBUG(dbgs() << " noted header: "
<< CI.getSSAContext().print(CycleHeader) << "\n");
@@ -1418,13 +1407,13 @@ void ModifiedPostOrder<ContextT>::computeCyclePO(
// Visit the header last
LLVM_DEBUG(dbgs() << " finishing header: "
<< CI.getSSAContext().print(CycleHeader) << "\n");
- appendBlock(*CycleHeader, CI.isReducible(*Cycle));
+ appendBlock(*CycleHeader, CI.isReducible(C));
// Initialize with immediate successors
for (auto *BB : successors(CycleHeader)) {
LLVM_DEBUG(dbgs() << " examine succ: " << CI.getSSAContext().print(BB)
<< "\n");
- if (!CI.contains(*Cycle, BB))
+ if (!CI.contains(C, BB))
continue;
if (BB == CycleHeader)
continue;
@@ -1436,7 +1425,7 @@ void ModifiedPostOrder<ContextT>::computeCyclePO(
}
// Compute PO inside region
- computeStackPO(Stack, CI, Cycle, Finalized);
+ computeStackPO(Stack, CI, C, Finalized);
LLVM_DEBUG(dbgs() << "exited computeCyclePO\n");
}
@@ -1449,7 +1438,7 @@ void llvm::ModifiedPostOrder<ContextT>::compute(const CycleInfoT &CI) {
auto *F = CI.getFunction();
Stack.reserve(24); // FIXME made-up number
Stack.push_back(&F->front());
- computeStackPO(Stack, CI, nullptr, Finalized);
+ computeStackPO(Stack, CI, Cycle(), Finalized);
}
} // namespace llvm
diff --git a/llvm/include/llvm/ADT/GenericUniformityInfo.h b/llvm/include/llvm/ADT/GenericUniformityInfo.h
index e8d0981778165..88d9377f4c3b4 100644
--- a/llvm/include/llvm/ADT/GenericUniformityInfo.h
+++ b/llvm/include/llvm/ADT/GenericUniformityInfo.h
@@ -38,10 +38,10 @@ template <typename ContextT> class GenericUniformityInfo {
using ThisT = GenericUniformityInfo<ContextT>;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleT = typename CycleInfoT::CycleT;
+ using Cycle = typename CycleInfoT::Cycle;
using TemporalDivergenceTuple =
- std::tuple<ConstValueRefT, InstructionT *, const CycleT *>;
+ std::tuple<ConstValueRefT, InstructionT *, Cycle>;
GenericUniformityInfo(const DominatorTreeT &DT, const CycleInfoT &CI,
const TargetTransformInfo *TTI = nullptr);
diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
index a9890b2fe00c0..e7f7b8f921beb 100644
--- a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
+++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
@@ -23,7 +23,7 @@
namespace llvm {
class MachineCycleInfo : public GenericCycleInfo<MachineSSAContext> {};
-using MachineCycle = MachineCycleInfo::CycleT;
+using MachineCycle = MachineCycleInfo::Cycle;
/// Legacy analysis pass which computes a \ref MachineCycleInfo.
class LLVM_ABI MachineCycleInfoWrapperPass : public MachineFunctionPass {
diff --git a/llvm/include/llvm/IR/CycleInfo.h b/llvm/include/llvm/IR/CycleInfo.h
index bf719fdfe9693..20f9acd03dc1a 100644
--- a/llvm/include/llvm/IR/CycleInfo.h
+++ b/llvm/include/llvm/IR/CycleInfo.h
@@ -23,7 +23,7 @@ namespace llvm {
// Use class instead of using to allow forward declarations.
class CycleInfo : public GenericCycleInfo<SSAContext> {};
-using Cycle = CycleInfo::CycleT;
+using Cycle = CycleInfo::Cycle;
} // namespace llvm
diff --git a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
index 19b03a0e581ff..0decda51915ae 100644
--- a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
+++ b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
@@ -132,7 +132,7 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
const auto &F = *Context.getFunction();
DenseMap<const BlockT *, SmallVector<const InstructionT *, 8>> LiveTokenMap;
- DenseMap<const CycleT *, const InstructionT *> CycleHearts;
+ DenseMap<Cycle, const InstructionT *> CycleHearts;
// Just like the DominatorTree, compute the CycleInfo locally so that we
// can run the verifier outside of a pass manager and we don't rely on
@@ -153,12 +153,12 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
// Check static rules about cycles.
auto *BB = User->getParent();
- auto *BBCycle = CI.getCycle(BB);
+ Cycle BBCycle = CI.getCycle(BB);
if (!BBCycle)
return;
auto *DefBB = Token->getParent();
- if (DefBB == BB || CI.contains(*BBCycle, DefBB)) {
+ if (DefBB == BB || CI.contains(BBCycle, DefBB)) {
// degenerate occurrence of a loop intrinsic
return;
}
@@ -170,13 +170,13 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
{Context.print(User), CI.print(BBCycle)});
while (true) {
- auto *Parent = CI.getParentCycle(*BBCycle);
- if (!Parent || CI.contains(*Parent, DefBB))
+ Cycle Parent = CI.getParentCycle(BBCycle);
+ if (!Parent || CI.contains(Parent, DefBB))
break;
BBCycle = Parent;
};
- Check(CI.isReducible(*BBCycle) && BB == CI.getHeader(*BBCycle),
+ Check(CI.isReducible(BBCycle) && BB == CI.getHeader(BBCycle),
"Cycle heart must dominate all blocks in the cycle.",
{Context.print(User), Context.printAsOperand(BB), CI.print(BBCycle)});
Check(!CycleHearts.count(BBCycle),
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index f4a57df755cb2..a8fb10b0c3637 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -178,10 +178,10 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
}
}
- SmallPtrSet<const Cycle *, 8> CyclesWithHoles;
+ DenseSet<Cycle> CyclesWithHoles;
if (CI && ExclusionSet) {
for (auto *BB : *ExclusionSet) {
- if (const Cycle *C = CI->getTopLevelParentCycle(BB))
+ if (Cycle C = CI->getTopLevelParentCycle(BB))
CyclesWithHoles.insert(C);
}
}
@@ -194,10 +194,10 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
}
}
- SmallPtrSet<const Cycle *, 2> StopCycles;
+ DenseSet<Cycle> StopCycles;
if (CI) {
for (auto *StopSetBB : StopSet) {
- if (const Cycle *C = CI->getTopLevelParentCycle(StopSetBB))
+ if (Cycle C = CI->getTopLevelParentCycle(StopSetBB))
StopCycles.insert(C);
}
}
@@ -232,12 +232,12 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
return true;
}
- const Cycle *OuterC = nullptr;
+ Cycle OuterC;
if (CI) {
OuterC = CI->getTopLevelParentCycle(BB);
if (OuterC) {
if (CyclesWithHoles.count(OuterC))
- OuterC = nullptr;
+ OuterC = Cycle();
else if (StopCycles.contains(OuterC))
return true;
} else {
@@ -264,7 +264,7 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
// ignoring any other blocks inside the loop body.
OuterL->getExitBlocks(Worklist);
} else if (OuterC) {
- CI->getExitBlocks(*OuterC, Worklist);
+ CI->getExitBlocks(OuterC, Worklist);
} else {
Worklist.append(succ_begin(BB), succ_end(BB));
}
@@ -358,7 +358,7 @@ bool llvm::isPotentiallyReachable(
// If cycle info is available, we can know for sure whether or not a
// block is part of a cycle.
if (CI)
- return CI->getCycle(BB) != nullptr;
+ return CI->getCycle(BB).isValid();
// If only loop info is available, even if the block is not part of a
// natural loop, it may still be part of an irreducible cycle.
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index 835d3efe7d4e9..ab6067db102ef 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -129,7 +129,7 @@ void llvm::GenericUniformityAnalysisImpl<
if (CI.contains(DefCycle, UserInstr->getParent()))
continue;
markDivergent(*UserInstr);
- recordTemporalDivergence(&I, UserInstr, &DefCycle);
+ recordTemporalDivergence(&I, UserInstr, DefCycle);
}
}
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 0e2ce14876183..77eae30e22398 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -258,11 +258,11 @@ class MachineSinking {
bool &BreakPHIEdge,
AllSuccsCache &AllSuccessors);
- void FindCycleSinkCandidates(MachineCycle *Cycle, MachineBasicBlock *BB,
+ void FindCycleSinkCandidates(MachineCycle Cycle, MachineBasicBlock *BB,
SmallVectorImpl<MachineInstr *> &Candidates);
bool
- aggressivelySinkIntoCycle(MachineCycle *Cycle, MachineInstr &I,
+ aggressivelySinkIntoCycle(MachineCycle Cycle, MachineInstr &I,
DenseMap<SinkItem, MachineInstr *> &SunkInstrs);
bool isProfitableToSinkTo(Register Reg, MachineInstr &MI,
@@ -720,7 +720,7 @@ static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI) {
}
void MachineSinking::FindCycleSinkCandidates(
- MachineCycle *Cycle, MachineBasicBlock *BB,
+ MachineCycle Cycle, MachineBasicBlock *BB,
SmallVectorImpl<MachineInstr *> &Candidates) {
for (auto &MI : *BB) {
LLVM_DEBUG(dbgs() << "CycleSink: Analysing candidate: " << MI);
@@ -733,7 +733,7 @@ void MachineSinking::FindCycleSinkCandidates(
"target\n");
continue;
}
- if (!isCycleInvariant(*CI, *Cycle, MI)) {
+ if (!isCycleInvariant(*CI, Cycle, MI)) {
LLVM_DEBUG(dbgs() << "CycleSink: Instruction is not cycle invariant\n");
continue;
}
@@ -883,7 +883,7 @@ bool MachineSinking::run(MachineFunction &MF) {
}
if (SinkInstsIntoCycle) {
- SmallVector<MachineCycle *, 8> Cycles(CI->toplevel_cycles());
+ SmallVector<MachineCycle, 8> Cycles(CI->toplevel_cycles());
SchedModel.init(STI);
bool HasHighPressure;
@@ -894,8 +894,8 @@ bool MachineSinking::run(MachineFunction &MF) {
++Stage, SunkInstrs.clear()) {
HasHighPressure = false;
- for (auto *Cycle : Cycles) {
- MachineBasicBlock *Preheader = CI->getCyclePreheader(*Cycle);
+ for (MachineCycle Cycle : Cycles) {
+ MachineBasicBlock *Preheader = CI->getCyclePreheader(Cycle);
if (!Preheader) {
LLVM_DEBUG(dbgs() << "CycleSink: Can't find preheader\n");
continue;
@@ -1116,12 +1116,12 @@ bool MachineSinking::isLegalToBreakCriticalEdge(MachineInstr &MI,
if (!SplitEdges || FromBB == ToBB || !FromBB->isSuccessor(ToBB))
return false;
- MachineCycle *FromCycle = CI->getCycle(FromBB);
- MachineCycle *ToCycle = CI->getCycle(ToBB);
+ MachineCycle FromCycle = CI->getCycle(FromBB);
+ MachineCycle ToCycle = CI->getCycle(ToBB);
// Check for backedges of more "complex" cycles.
if (FromCycle == ToCycle && FromCycle &&
- (!CI->isReducible(*FromCycle) || CI->getHeader(*FromCycle) == ToBB))
+ (!CI->isReducible(FromCycle) || CI->getHeader(FromCycle) == ToBB))
return false;
// It's not always legal to break critical edges and sink the computation
@@ -1304,7 +1304,7 @@ bool MachineSinking::isProfitableToSinkTo(Register Reg, MachineInstr &MI,
FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
- MachineCycle *MCycle = CI->getCycle(MBB);
+ MachineCycle MCycle = CI->getCycle(MBB);
// If the instruction is not inside a cycle, it is not profitable to sink MI
// to a post dominate block SuccToSinkTo.
@@ -1340,14 +1340,14 @@ bool MachineSinking::isProfitableToSinkTo(Register Reg, MachineInstr &MI,
MachineInstr *DefMI = MRI->getVRegDef(Reg);
if (!DefMI)
continue;
- MachineCycle *Cycle = CI->getCycle(DefMI->getParent());
+ MachineCycle Cycle = CI->getCycle(DefMI->getParent());
// DefMI is defined outside of cycle. There should be no live range
// impact for this operand. Defination outside of cycle means:
// 1: defination is outside of cycle.
// 2: defination is in this cycle, but it is a PHI in the cycle header.
if (Cycle != MCycle ||
- (DefMI->isPHI() && Cycle && CI->isReducible(*Cycle) &&
- CI->getHeader(*Cycle) == DefMI->getParent()))
+ (DefMI->isPHI() && Cycle && CI->isReducible(Cycle) &&
+ CI->getHeader(Cycle) == DefMI->getParent()))
continue;
// The DefMI is defined inside the cycle.
// If sinking this operand makes some register pressure set exceed limit,
@@ -1752,14 +1752,14 @@ bool MachineSinking::hasStoreBetween(MachineBasicBlock *From,
/// based on the amount of sinking, or the type of ops being sunk (so long as
/// they are safe to sink).
bool MachineSinking::aggressivelySinkIntoCycle(
- MachineCycle *Cycle, MachineInstr &I,
+ MachineCycle Cycle, MachineInstr &I,
DenseMap<SinkItem, MachineInstr *> &SunkInstrs) {
// TODO: support instructions with multiple defs
if (I.getNumDefs() > 1)
return false;
LLVM_DEBUG(dbgs() << "AggressiveCycleSink: Finding sink block for: " << I);
- assert(CI->getCyclePreheader(*Cycle) && "Cycle sink needs a preheader block");
+ assert(CI->getCyclePreheader(Cycle) && "Cycle sink needs a preheader block");
SmallVector<std::pair<RegSubRegPair, MachineInstr *>> Uses;
MachineOperand &DefMO = I.getOperand(0);
@@ -1781,7 +1781,7 @@ bool MachineSinking::aggressivelySinkIntoCycle(
"can't sink.\n");
continue;
}
- if (!CI->contains(*Cycle, MI->getParent())) {
+ if (!CI->contains(Cycle, MI->getParent())) {
LLVM_DEBUG(
dbgs() << "AggressiveCycleSink: Use not in cycle, can't sink.\n");
continue;
@@ -1914,8 +1914,8 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
// Don't sink instructions into a cycle.
if (!TryBreak && CI->getCycle(SuccToSinkTo) &&
- (!CI->isReducible(*CI->getCycle(SuccToSinkTo)) ||
- CI->getHeader(*CI->getCycle(SuccToSinkTo)) == SuccToSinkTo)) {
+ (!CI->isReducible(CI->getCycle(SuccToSinkTo)) ||
+ CI->getHeader(CI->getCycle(SuccToSinkTo)) == SuccToSinkTo)) {
LLVM_DEBUG(dbgs() << " *** NOTE: cycle header found\n");
TryBreak = true;
}
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index 8b5dd5dae44b3..ea055a3d44431 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -142,7 +142,7 @@ void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::
continue;
markDivergent(UserInstr);
- recordTemporalDivergence(Reg, &UserInstr, &DefCycle);
+ recordTemporalDivergence(Reg, &UserInstr, DefCycle);
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
index 538bc72102e3f..55ca311886c92 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -240,15 +240,15 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
// In case of use outside muliple nested cycles or muliple uses we only need
// to merge lane mask across largest relevant cycle.
- SmallDenseMap<Register, std::pair<const MachineCycle *, Register>> LRCCache;
+ SmallDenseMap<Register, std::pair<MachineCycle, Register>> LRCCache;
for (auto [Reg, UseInst, LRC] : MUI->getTemporalDivergenceList()) {
if (MRI->getType(Reg) != LLT::scalar(1))
continue;
auto [LRCCacheIter, RegNotCached] = LRCCache.try_emplace(Reg);
auto &CycleMergedMask = LRCCacheIter->getSecond();
- const MachineCycle *&CachedLRC = CycleMergedMask.first;
- if (RegNotCached || CInfo.contains(*LRC, *CachedLRC)) {
+ MachineCycle &CachedLRC = CycleMergedMask.first;
+ if (RegNotCached || CInfo.contains(LRC, CachedLRC)) {
CachedLRC = LRC;
}
}
@@ -256,7 +256,7 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
for (auto &LRCCacheEntry : LRCCache) {
Register Reg = LRCCacheEntry.first;
auto &CycleMergedMask = LRCCacheEntry.getSecond();
- const MachineCycle *Cycle = CycleMergedMask.first;
+ MachineCycle Cycle = CycleMergedMask.first;
Register MergedMask = MRI->createVirtualRegister(BoolS1);
SSAUpdater.Initialize(MergedMask);
@@ -264,9 +264,9 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
MachineBasicBlock *MBB = MRI->getVRegDef(Reg)->getParent();
SSAUpdater.AddAvailableValue(MBB, MergedMask);
- for (auto Entry : CInfo.getEntries(*Cycle)) {
+ for (auto Entry : CInfo.getEntries(Cycle)) {
for (MachineBasicBlock *Pred : Entry->predecessors()) {
- if (!CInfo.contains(*Cycle, Pred)) {
+ if (!CInfo.contains(Cycle, Pred)) {
B.setInsertPt(*Pred, Pred->getFirstTerminator());
auto ImplDef = B.buildInstr(AMDGPU::IMPLICIT_DEF, {BoolS1}, {});
SSAUpdater.AddAvailableValue(Pred, ImplDef.getReg(0));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2395c326dc3fc..c1aa477576e4f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -228,16 +228,16 @@ bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
MachineInstr *SgprDef = MRI.getVRegDef(Op.getReg());
// SgprDef defined inside cycle
- MachineCycle *FromCycle = CI->getCycle(SgprDef->getParent());
- if (FromCycle == nullptr)
+ MachineCycle FromCycle = CI->getCycle(SgprDef->getParent());
+ if (!FromCycle)
continue;
- MachineCycle *ToCycle = CI->getCycle(SuccToSinkTo);
+ MachineCycle ToCycle = CI->getCycle(SuccToSinkTo);
// Check if there is a FromCycle that contains SgprDef's basic block but
// does not contain SuccToSinkTo and also has divergent exit condition.
- while (FromCycle && !(ToCycle && CI->contains(*FromCycle, *ToCycle))) {
+ while (FromCycle && !(ToCycle && CI->contains(FromCycle, ToCycle))) {
SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
- CI->getExitingBlocks(*FromCycle, ExitingBlocks);
+ CI->getExitingBlocks(FromCycle, ExitingBlocks);
// FromCycle has divergent exit condition.
for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
@@ -245,7 +245,7 @@ bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
return false;
}
- FromCycle = CI->getParentCycle(*FromCycle);
+ FromCycle = CI->getParentCycle(FromCycle);
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index cde821bfb8ba8..4c5b341db0e83 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -67,7 +67,7 @@ class SILowerSGPRSpills {
MBBVector SaveBlocks;
MBBVector RestoreBlocks;
- MachineBasicBlock *getCycleDomBB(MachineCycle *C);
+ MachineBasicBlock *getCycleDomBB(MachineCycle C);
public:
SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
@@ -302,17 +302,17 @@ bool SILowerSGPRSpills::spillCalleeSavedRegs(
return false;
}
-MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(MachineCycle *C) {
+MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(MachineCycle C) {
// If the insertion point lands on a cycle entry, move it to a block that
// dominates all entries.
- if (MCI->isReducible(*C)) {
- if (auto *IDom = MDT->getNode(MCI->getHeader(*C))->getIDom())
+ if (MCI->isReducible(C)) {
+ if (auto *IDom = MDT->getNode(MCI->getHeader(C))->getIDom())
return IDom->getBlock();
llvm_unreachable("Expected cycle to have an IDom.");
return nullptr;
}
- ArrayRef<MachineBasicBlock *> Entries = MCI->getEntries(*C);
+ ArrayRef<MachineBasicBlock *> Entries = MCI->getEntries(C);
assert(!Entries.empty() && "Expected cycle to have at least one entry.");
MachineBasicBlock *EntryBB = Entries[0];
for (unsigned I = 1; I < Entries.size(); ++I)
@@ -518,7 +518,7 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
LaneVGPRInsertPt IP = LaneVGPRDomInstr[Reg];
- if (MachineCycle *C = MCI->getTopLevelParentCycle(IP.MBB)) {
+ if (MachineCycle C = MCI->getTopLevelParentCycle(IP.MBB)) {
MachineBasicBlock *AdjMBB = getCycleDomBB(C);
IP = insertPt(AdjMBB, AdjMBB->getFirstTerminator());
}
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index fb70ee86920ca..38bf9df607b48 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -214,16 +214,16 @@ ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
} // namespace llvm
static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
- bool HeaderOnly, Cycle **CPtr = nullptr) {
+ bool HeaderOnly, Cycle *CPtr = nullptr) {
if (!CI)
return true;
auto *BB = I->getParent();
- auto *C = CI->getCycle(BB);
+ Cycle C = CI->getCycle(BB);
if (!C)
return false;
if (CPtr)
*CPtr = C;
- return !HeaderOnly || BB == CI->getHeader(*C);
+ return !HeaderOnly || BB == CI->getHeader(C);
}
/// Checks if a type could have padding bytes.
@@ -11421,7 +11421,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl {
A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
*PHI.getFunction());
- Cycle *C = nullptr;
+ Cycle C;
bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
@@ -11437,7 +11437,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl {
// If the incoming value is not the PHI but an instruction in the same
// cycle we might have multiple versions of it flying around.
if (CyclePHI && isa<Instruction>(V) &&
- (!C || CI->contains(*C, cast<Instruction>(V)->getParent())))
+ (!C || CI->contains(C, cast<Instruction>(V)->getParent())))
return false;
Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 6bc77b7583b24..9ff8801181602 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1620,7 +1620,7 @@ bool DSEState::isGuaranteedLoopIndependent(const Instruction *Current,
// would also be valid but we currently disable that to limit compile time).
if (Current->getParent() == KillingDef->getParent())
return true;
- const Cycle *CurrentC = CI.getCycle(Current->getParent());
+ Cycle CurrentC = CI.getCycle(Current->getParent());
if (CurrentC && CurrentC == CI.getCycle(KillingDef->getParent()))
return true;
// Otherwise check the memory location is invariant to any loops.
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index ea4f468418566..dd1c4723d9b4d 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -718,18 +718,17 @@ static bool updateCycleLoopInfo(TI *LCI, BasicBlock *CallBrBlock,
if (!LCI)
return false;
- T *LC;
- if constexpr (std::is_same_v<TI, CycleInfo>)
- LC = LCI->getSmallestCommonCycle(CallBrBlock, Succ);
- else
- LC = LCI->getSmallestCommonLoop(CallBrBlock, Succ);
- if (!LC)
- return false;
-
- if constexpr (std::is_same_v<TI, CycleInfo>)
+ if constexpr (std::is_same_v<TI, CycleInfo>) {
+ T LC = LCI->getSmallestCommonCycle(CallBrBlock, Succ);
+ if (!LC)
+ return false;
LCI->addBlockToCycle(CallBrTarget, LC);
- else
+ } else {
+ T *LC = LCI->getSmallestCommonLoop(CallBrBlock, Succ);
+ if (!LC)
+ return false;
LC->addBasicBlockToLoop(CallBrTarget, *LCI);
+ }
return true;
}
diff --git a/llvm/lib/Transforms/Utils/FixIrreducible.cpp b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
index b6bed8413c3fe..63940b3777935 100644
--- a/llvm/lib/Transforms/Utils/FixIrreducible.cpp
+++ b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
@@ -220,7 +220,7 @@ static void reconnectChildLoops(LoopInfo &LI, Loop *ParentLoop, Loop *NewLoop,
}
}
-static void updateLoopInfo(CycleInfo &CI, LoopInfo &LI, Cycle &C,
+static void updateLoopInfo(CycleInfo &CI, LoopInfo &LI, Cycle C,
ArrayRef<BasicBlock *> GuardBlocks) {
// The parent loop is a natural loop L mapped to the cycle header H as long as
// H is not also the header of L. In the latter case, L is destroyed and we
@@ -274,11 +274,11 @@ static void updateLoopInfo(CycleInfo &CI, LoopInfo &LI, Cycle &C,
// Given a set of blocks and headers in an irreducible SCC, convert it into a
// natural loop. Also insert this new loop at its appropriate place in the
// hierarchy of loops.
-static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
+static bool fixIrreducible(Cycle C, CycleInfo &CI, DominatorTree &DT,
LoopInfo *LI) {
if (CI.isReducible(C))
return false;
- LLVM_DEBUG(dbgs() << "Processing cycle:\n" << CI.print(&C) << "\n";);
+ LLVM_DEBUG(dbgs() << "Processing cycle:\n" << CI.print(C) << "\n";);
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
ControlFlowHub CHub;
@@ -411,13 +411,13 @@ static bool fixIrreducible(Cycle &C, CycleInfo &CI, DominatorTree &DT,
for (auto *G : GuardBlocks) {
LLVM_DEBUG(dbgs() << "added guard block to cycle: " << G->getName()
<< "\n");
- CI.addBlockToCycle(G, &C);
+ CI.addBlockToCycle(G, C);
}
CI.setSingleEntry(C, GuardBlocks[0]);
CI.verifyCycle(C);
- if (Cycle *Parent = CI.getParentCycle(C))
- CI.verifyCycle(*Parent);
+ if (Cycle Parent = CI.getParentCycle(C))
+ CI.verifyCycle(Parent);
LLVM_DEBUG(dbgs() << "Finished one cycle:\n"; CI.print(dbgs()););
return true;
@@ -429,7 +429,7 @@ static bool FixIrreducibleImpl(Function &F, CycleInfo &CI, DominatorTree &DT,
<< F.getName() << "\n");
bool Changed = false;
- for (Cycle &C : CI.cycles())
+ for (Cycle C : CI.cycles())
Changed |= fixIrreducible(C, CI, DT, LI);
if (!Changed)
>From 286b334c24a05e83c42f164ac363853293d79fd6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Thu, 16 Jul 2026 10:38:30 -0700
Subject: [PATCH 2/6] const Cycle & -> Cycle
---
llvm/include/llvm/ADT/GenericUniformityImpl.h | 5 ++---
llvm/lib/Analysis/UniformityAnalysis.cpp | 4 ++--
llvm/lib/CodeGen/MachineUniformityAnalysis.cpp | 8 ++++----
3 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 34f53084fdf7f..16279bd5bf061 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -493,14 +493,13 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
void analyzeCycleExitDivergence(Cycle DefCycle);
/// \brief Mark as divergent all uses of \p I that are outside \p DefCycle.
- void propagateTemporalDivergence(const InstructionT &I,
- const Cycle &DefCycle);
+ void propagateTemporalDivergence(const InstructionT &I, Cycle DefCycle);
/// \brief Push all users of \p Val (in the region) to the worklist.
void pushUsers(const InstructionT &I);
void pushUsers(ConstValueRefT V);
- bool usesValueFromCycle(const InstructionT &I, const Cycle &DefCycle) const;
+ bool usesValueFromCycle(const InstructionT &I, Cycle DefCycle) const;
/// \brief Whether \p Def is divergent when read in \p ObservingBlock.
bool isTemporalDivergent(const BlockT &ObservingBlock,
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index ab6067db102ef..7a2c2139f69ce 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -109,7 +109,7 @@ template <> void llvm::GenericUniformityAnalysisImpl<SSAContext>::initialize() {
template <>
bool llvm::GenericUniformityAnalysisImpl<SSAContext>::usesValueFromCycle(
- const Instruction &I, const Cycle &DefCycle) const {
+ const Instruction &I, Cycle DefCycle) const {
assert(!isAlwaysUniform(I));
for (const Use &U : I.operands()) {
if (auto *I = dyn_cast<Instruction>(&U)) {
@@ -123,7 +123,7 @@ bool llvm::GenericUniformityAnalysisImpl<SSAContext>::usesValueFromCycle(
template <>
void llvm::GenericUniformityAnalysisImpl<
SSAContext>::propagateTemporalDivergence(const Instruction &I,
- const Cycle &DefCycle) {
+ Cycle DefCycle) {
for (auto *User : I.users()) {
auto *UserInstr = cast<Instruction>(User);
if (CI.contains(DefCycle, UserInstr->getParent()))
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index ea055a3d44431..49cf1a8920087 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -109,7 +109,7 @@ void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::pushUsers(
template <>
bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::usesValueFromCycle(
- const MachineInstr &I, const MachineCycle &DefCycle) const {
+ const MachineInstr &I, MachineCycle DefCycle) const {
assert(!isAlwaysUniform(I));
for (auto &Op : I.operands()) {
if (!Op.isReg() || !Op.readsReg())
@@ -129,9 +129,9 @@ bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::usesValueFromCycle(
}
template <>
-void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::
- propagateTemporalDivergence(const MachineInstr &I,
- const MachineCycle &DefCycle) {
+void llvm::GenericUniformityAnalysisImpl<
+ MachineSSAContext>::propagateTemporalDivergence(const MachineInstr &I,
+ MachineCycle DefCycle) {
const auto &RegInfo = F.getRegInfo();
for (auto &Op : I.all_defs()) {
if (!Op.getReg().isVirtual())
>From 7a41b5313908398ca555926336eaf34917b70664 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Thu, 16 Jul 2026 18:30:23 -0700
Subject: [PATCH 3/6] Cycle -> CycleRef; de-template
---
.../llvm/ADT/GenericConvergenceVerifier.h | 2 +-
llvm/include/llvm/ADT/GenericCycleImpl.h | 36 ++---
llvm/include/llvm/ADT/GenericCycleInfo.h | 113 +++++++-------
llvm/include/llvm/ADT/GenericUniformityImpl.h | 141 +++++++++---------
llvm/include/llvm/ADT/GenericUniformityInfo.h | 4 +-
.../llvm/CodeGen/MachineCycleAnalysis.h | 3 +-
llvm/include/llvm/IR/CycleInfo.h | 2 -
.../llvm/IR/GenericConvergenceVerifierImpl.h | 6 +-
llvm/lib/Analysis/CFG.cpp | 12 +-
llvm/lib/Analysis/UniformityAnalysis.cpp | 4 +-
llvm/lib/CodeGen/MachineCycleAnalysis.cpp | 4 +-
llvm/lib/CodeGen/MachineSink.cpp | 20 +--
.../lib/CodeGen/MachineUniformityAnalysis.cpp | 4 +-
.../AMDGPUGlobalISelDivergenceLowering.cpp | 6 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 4 +-
llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp | 6 +-
.../Transforms/IPO/AttributorAttributes.cpp | 6 +-
.../Scalar/DeadStoreElimination.cpp | 2 +-
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 3 +-
llvm/lib/Transforms/Utils/FixIrreducible.cpp | 8 +-
20 files changed, 194 insertions(+), 192 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
index 138f5d24be109..3ce490d43acdc 100644
--- a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
+++ b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
@@ -28,7 +28,7 @@ template <typename ContextT> class GenericConvergenceVerifier {
using InstructionT = typename ContextT::InstructionT;
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using Cycle = typename CycleInfoT::Cycle;
+ using CycleRef = typename CycleInfoT::CycleRef;
void initialize(raw_ostream *OS,
function_ref<void(const Twine &Message)> FailureCB,
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 6e60282581442..092f7bd16a71f 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -36,7 +36,7 @@ namespace llvm {
template <typename ContextT>
void GenericCycleInfo<ContextT>::getExitBlocks(
- Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const {
+ CycleRef C, SmallVectorImpl<BlockT *> &TmpStorage) const {
if (ExitBlocksCaches.empty())
ExitBlocksCaches.resize(NumCycles);
auto &Cache = ExitBlocksCaches[C.Index];
@@ -66,7 +66,7 @@ void GenericCycleInfo<ContextT>::getExitBlocks(
template <typename ContextT>
void GenericCycleInfo<ContextT>::getExitingBlocks(
- Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const {
+ CycleRef C, SmallVectorImpl<BlockT *> &TmpStorage) const {
for (BlockT *Block : getBlocks(C)) {
for (BlockT *Succ : successors(Block)) {
if (!contains(C, Succ)) {
@@ -78,7 +78,8 @@ void GenericCycleInfo<ContextT>::getExitingBlocks(
}
template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getCyclePreheader(Cycle C) const -> BlockT * {
+auto GenericCycleInfo<ContextT>::getCyclePreheader(CycleRef C) const
+ -> BlockT * {
BlockT *Predecessor = getCyclePredecessor(C);
if (!Predecessor)
return nullptr;
@@ -96,7 +97,7 @@ auto GenericCycleInfo<ContextT>::getCyclePreheader(Cycle C) const -> BlockT * {
}
template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getCyclePredecessor(Cycle C) const
+auto GenericCycleInfo<ContextT>::getCyclePredecessor(CycleRef C) const
-> BlockT * {
if (!isReducible(C))
return nullptr;
@@ -117,7 +118,7 @@ auto GenericCycleInfo<ContextT>::getCyclePredecessor(Cycle C) const
}
template <typename ContextT>
-void GenericCycleInfo<ContextT>::verifyCycle(Cycle C) const {
+void GenericCycleInfo<ContextT>::verifyCycle(CycleRef C) const {
#ifndef NDEBUG
assert(getNumBlocks(C) != 0 && "Cycle cannot be empty.");
DenseSet<BlockT *> Blocks;
@@ -191,11 +192,11 @@ void GenericCycleInfo<ContextT>::verifyCycle(Cycle C) const {
}
template <typename ContextT>
-void GenericCycleInfo<ContextT>::verifyCycleNest(Cycle C) const {
+void GenericCycleInfo<ContextT>::verifyCycleNest(CycleRef C) const {
#ifndef NDEBUG
const CycleT &Cyc = deref(C);
// Check the subcycles.
- for (Cycle Child : children(C)) {
+ for (auto Child : children(C)) {
// Each block in each subcycle should be contained within this cycle.
for (BlockT *BB : getBlocks(Child)) {
assert(contains(C, BB) &&
@@ -297,7 +298,7 @@ void GenericCycleInfo<ContextT>::addToBlockMap(BlockT *Block, CycleT *Cycle) {
}
template <typename ContextT>
-void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, Cycle C) {
+void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleRef C) {
CycleT &Cyc = deref(C);
// Make sure BlockMap is large enough for the new block.
unsigned Number = GraphTraits<BlockT *>::getNumber(Block);
@@ -587,7 +588,7 @@ void GenericCycleInfo<ContextT>::splitCriticalEdge(BlockT *Pred, BlockT *Succ,
BlockT *NewBlock) {
// Edge Pred-Succ is replaced by edges Pred-NewBlock and NewBlock-Succ, all
// cycles that had blocks Pred and Succ also get NewBlock.
- Cycle C = getSmallestCommonCycle(getCycle(Pred), getCycle(Succ));
+ CycleRef C = getSmallestCommonCycle(getCycle(Pred), getCycle(Succ));
if (!C)
return;
@@ -600,10 +601,11 @@ void GenericCycleInfo<ContextT>::splitCriticalEdge(BlockT *Pred, BlockT *Succ,
/// \returns the innermost cycle containing both \p A and \p B
/// or nullptr if there is no such cycle.
template <typename ContextT>
-auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(Cycle A, Cycle B) const
- -> Cycle {
+auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(CycleRef A,
+ CycleRef B) const
+ -> CycleRef {
if (!A || !B)
- return Cycle();
+ return CycleRef();
// If cycles A and B have different depth replace them with parent cycle
// until they have the same depth.
@@ -630,7 +632,7 @@ auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(Cycle A, Cycle B) const
template <typename ContextT>
auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(BlockT *A,
BlockT *B) const
- -> Cycle {
+ -> CycleRef {
return getSmallestCommonCycle(getCycle(A), getCycle(B));
}
@@ -643,7 +645,7 @@ void GenericCycleInfo<ContextT>::verifyCycleNest(bool VerifyFull) const {
#ifndef NDEBUG
DenseSet<BlockT *> CycleHeaders;
- for (Cycle C : cycles()) {
+ for (auto C : cycles()) {
BlockT *Header = getHeader(C);
assert(CycleHeaders.insert(Header).second);
if (VerifyFull)
@@ -652,7 +654,7 @@ void GenericCycleInfo<ContextT>::verifyCycleNest(bool VerifyFull) const {
verifyCycleNest(C);
// Check the block map entries for blocks contained in this cycle.
for (BlockT *BB : getBlocks(C)) {
- Cycle InBlockMap = getCycle(BB);
+ CycleRef InBlockMap = getCycle(BB);
assert(InBlockMap.isValid());
assert(contains(C, InBlockMap));
}
@@ -668,7 +670,7 @@ template <typename ContextT> void GenericCycleInfo<ContextT>::verify() const {
/// \brief Print the cycle info.
template <typename ContextT>
void GenericCycleInfo<ContextT>::print(raw_ostream &Out) const {
- for (Cycle C : cycles()) {
+ for (auto C : cycles()) {
for (unsigned I = 0, Depth = getDepth(C); I < Depth; ++I)
Out << " ";
@@ -678,7 +680,7 @@ void GenericCycleInfo<ContextT>::print(raw_ostream &Out) const {
/// \brief Print a single cycle: its depth, entries, and remaining blocks.
template <typename ContextT>
-Printable GenericCycleInfo<ContextT>::print(Cycle C) const {
+Printable GenericCycleInfo<ContextT>::print(CycleRef C) const {
return Printable([this, C](raw_ostream &Out) {
Out << "depth=" << getDepth(C) << ": entries(" << printEntries(C, Context)
<< ')';
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 7fc8b5706cec4..53f744f082038 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -97,34 +97,32 @@ template <typename ContextT> class GenericCycle {
/// preorder index; a default-constructed handle is invalid ("no cycle"). All
/// queries live on GenericCycleInfo, which resolves the handle to storage.
///
-/// Handles remain valid as long as the cycle forest is not recomputed.
-/// addBlockToCycle() adds a block but never adds, removes, or reorders cycles,
-/// so it leaves every handle valid.
-template <typename ContextT> class GenericCycleRef {
+/// The handle is context-free: IR and machine cycles share one type, which
+/// keeps forward declarations simple. Handles remain valid as long as the
+/// cycle forest is not recomputed; addBlockToCycle() adds a block but never
+/// adds, removes, or reorders cycles, so it leaves every handle valid.
+class CycleRef {
static constexpr unsigned InvalidIndex = ~0u;
unsigned Index = InvalidIndex;
- explicit GenericCycleRef(unsigned Index) : Index(Index) {}
- friend class GenericCycleInfo<ContextT>;
- friend struct DenseMapInfo<GenericCycleRef<ContextT>>;
+ explicit CycleRef(unsigned Index) : Index(Index) {}
+ template <typename ContextT> friend class GenericCycleInfo;
+ friend struct DenseMapInfo<CycleRef>;
public:
- GenericCycleRef() = default;
+ CycleRef() = default;
bool isValid() const { return Index != InvalidIndex; }
explicit operator bool() const { return isValid(); }
- bool operator==(GenericCycleRef O) const { return Index == O.Index; }
- bool operator!=(GenericCycleRef O) const { return Index != O.Index; }
+ bool operator==(CycleRef O) const { return Index == O.Index; }
+ bool operator!=(CycleRef O) const { return Index != O.Index; }
};
-/// The empty/tombstone keys are distinct from the invalid handle, so an invalid
-/// handle stays a legal key -- matching the pointer world where nullptr is a
-/// legal DenseMap/SmallPtrSet key.
-template <typename ContextT> struct DenseMapInfo<GenericCycleRef<ContextT>> {
- using T = GenericCycleRef<ContextT>;
- static T getEmptyKey() { return T(~0u - 1); }
- static T getTombstoneKey() { return T(~0u - 2); }
- static unsigned getHashValue(T C) { return C.Index; }
- static bool isEqual(T A, T B) { return A.Index == B.Index; }
+/// DenseMap tracks bucket occupancy with a separate bitmap rather than sentinel
+/// key values, so only hashing and equality are needed and the invalid handle
+/// is itself a legal key.
+template <> struct DenseMapInfo<CycleRef> {
+ static unsigned getHashValue(CycleRef C) { return C.Index; }
+ static bool isEqual(CycleRef A, CycleRef B) { return A.Index == B.Index; }
};
/// \brief Cycle information for a function.
@@ -134,7 +132,7 @@ template <typename ContextT> class GenericCycleInfo {
/// The internal, by-value storage type for a cycle.
using CycleT = GenericCycle<ContextT>;
/// The opaque handle by which consumers refer to a cycle.
- using Cycle = GenericCycleRef<ContextT>;
+ using CycleRef = ::llvm::CycleRef;
using FunctionT = typename ContextT::FunctionT;
template <typename> friend class GenericCycleInfoCompute;
@@ -164,16 +162,16 @@ template <typename ContextT> class GenericCycleInfo {
/// Resolve a handle to its stored cycle. The assert catches deref of an
/// invalid handle and (partially) of a handle from another CycleInfo.
- CycleT &deref(Cycle C) {
+ CycleT &deref(CycleRef C) {
assert(C.Index < NumCycles);
return Cycles[C.Index];
}
- const CycleT &deref(Cycle C) const {
+ const CycleT &deref(CycleRef C) const {
assert(C.Index < NumCycles);
return Cycles[C.Index];
}
/// The handle for a stored cycle.
- Cycle ref(const CycleT &C) const { return Cycle(getCycleIndex(C)); }
+ CycleRef ref(const CycleT &C) const { return CycleRef(getCycleIndex(C)); }
/// The innermost cycle containing \p Block as a raw pointer, or null. Used
/// internally and during construction, where handles are not yet meaningful
@@ -197,7 +195,7 @@ template <typename ContextT> class GenericCycleInfo {
/// sibling follows the previous child's subtree.
struct const_child_iterator
: iterator_facade_base<const_child_iterator, std::forward_iterator_tag,
- Cycle, std::ptrdiff_t, Cycle, Cycle> {
+ CycleRef, std::ptrdiff_t, CycleRef, CycleRef> {
const GenericCycleInfo *CI = nullptr;
unsigned Index = 0;
@@ -205,7 +203,7 @@ template <typename ContextT> class GenericCycleInfo {
const_child_iterator(const GenericCycleInfo &CI, unsigned Index)
: CI(&CI), Index(Index) {}
- Cycle operator*() const { return Cycle(Index); }
+ CycleRef operator*() const { return CycleRef(Index); }
const_child_iterator &operator++() {
Index += 1 + CI->Cycles[Index].NumDescendants;
return *this;
@@ -218,13 +216,13 @@ template <typename ContextT> class GenericCycleInfo {
/// Sequential iteration over all cycles in forest preorder, yielding handles.
struct const_cycle_iterator
: iterator_facade_base<const_cycle_iterator, std::forward_iterator_tag,
- Cycle, std::ptrdiff_t, Cycle, Cycle> {
+ CycleRef, std::ptrdiff_t, CycleRef, CycleRef> {
unsigned Index = 0;
const_cycle_iterator() = default;
explicit const_cycle_iterator(unsigned Index) : Index(Index) {}
- Cycle operator*() const { return Cycle(Index); }
+ CycleRef operator*() const { return CycleRef(Index); }
const_cycle_iterator &operator++() {
++Index;
return *this;
@@ -255,45 +253,45 @@ template <typename ContextT> class GenericCycleInfo {
///
/// \returns the innermost cycle containing \p Block or an invalid handle if
/// it is not contained in any cycle.
- Cycle getCycle(const BlockT *Block) const {
+ CycleRef getCycle(const BlockT *Block) const {
CycleT *C = getCyclePtr(Block);
- return C ? ref(*C) : Cycle();
+ return C ? ref(*C) : CycleRef();
}
- BlockT *getHeader(Cycle C) const { return deref(C).Entries[0]; }
- bool isReducible(Cycle C) const { return deref(C).Entries.size() == 1; }
- Cycle getParentCycle(Cycle C) const {
+ BlockT *getHeader(CycleRef C) const { return deref(C).Entries[0]; }
+ bool isReducible(CycleRef C) const { return deref(C).Entries.size() == 1; }
+ CycleRef getParentCycle(CycleRef C) const {
CycleT *P = deref(C).ParentCycle;
- return P ? ref(*P) : Cycle();
+ return P ? ref(*P) : CycleRef();
}
- unsigned getDepth(Cycle C) const { return deref(C).Depth; }
- size_t getNumBlocks(Cycle C) const {
+ unsigned getDepth(CycleRef C) const { return deref(C).Depth; }
+ size_t getNumBlocks(CycleRef C) const {
const CycleT &Cyc = deref(C);
return Cyc.IdxEnd - Cyc.IdxBegin;
}
- ArrayRef<BlockT *> getEntries(Cycle C) const { return deref(C).Entries; }
- bool isEntry(Cycle C, const BlockT *Block) const {
+ ArrayRef<BlockT *> getEntries(CycleRef C) const { return deref(C).Entries; }
+ bool isEntry(CycleRef C, const BlockT *Block) const {
return is_contained(deref(C).Entries, Block);
}
- void setSingleEntry(Cycle C, BlockT *Block) {
+ void setSingleEntry(CycleRef C, BlockT *Block) {
auto &Entries = deref(C).Entries;
Entries.clear();
Entries.push_back(Block);
}
/// Returns true iff \p Outer contains \p Inner. O(1). Non-strict.
- bool contains(Cycle Outer, Cycle Inner) const {
+ bool contains(CycleRef Outer, CycleRef Inner) const {
const CycleT &O = deref(Outer);
const CycleT &I = deref(Inner);
return O.IdxBegin <= I.IdxBegin && I.IdxEnd <= O.IdxEnd;
}
- iterator_range<const_child_iterator> children(Cycle C) const {
+ iterator_range<const_child_iterator> children(CycleRef C) const {
unsigned First = C.Index + 1;
return llvm::make_range(
const_child_iterator(*this, First),
const_child_iterator(*this, First + deref(C).NumDescendants));
}
- Printable printEntries(Cycle C, const ContextT &Ctx) const {
+ Printable printEntries(CycleRef C, const ContextT &Ctx) const {
return Printable([this, C, &Ctx](raw_ostream &Out) {
ListSeparator LS(" ");
for (auto *Entry : deref(C).Entries)
@@ -302,32 +300,32 @@ template <typename ContextT> class GenericCycleInfo {
}
/// \brief Return whether \p Block is contained in \p C. O(1).
- bool contains(Cycle C, const BlockT *Block) const {
- Cycle Inner = getCycle(Block);
+ bool contains(CycleRef C, const BlockT *Block) const {
+ CycleRef Inner = getCycle(Block);
return Inner.isValid() && contains(C, Inner);
}
/// \brief Return the blocks of \p C, including those of nested cycles.
- ArrayRef<BlockT *> getBlocks(Cycle C) const {
+ ArrayRef<BlockT *> getBlocks(CycleRef C) const {
const CycleT &Cyc = deref(C);
return ArrayRef<BlockT *>(BlockLayout.begin() + Cyc.IdxBegin,
BlockLayout.begin() + Cyc.IdxEnd);
}
- Cycle getSmallestCommonCycle(Cycle A, Cycle B) const;
- Cycle getSmallestCommonCycle(BlockT *A, BlockT *B) const;
+ CycleRef getSmallestCommonCycle(CycleRef A, CycleRef B) const;
+ CycleRef getSmallestCommonCycle(BlockT *A, BlockT *B) const;
/// \brief Return the depth of the innermost cycle containing \p Block, or 0
/// if it is not contained in any cycle.
unsigned getCycleDepth(const BlockT *Block) const {
- Cycle C = getCycle(Block);
+ CycleRef C = getCycle(Block);
return C.isValid() ? getDepth(C) : 0;
}
- Cycle getTopLevelParentCycle(const BlockT *Block) const {
+ CycleRef getTopLevelParentCycle(const BlockT *Block) const {
CycleT *C = getCyclePtr(Block);
if (!C)
- return Cycle();
+ return CycleRef();
while (C->ParentCycle)
C = C->ParentCycle;
return ref(*C);
@@ -335,33 +333,34 @@ template <typename ContextT> class GenericCycleInfo {
/// Return all of the successor blocks of \p C: the blocks outside of \p C
/// which are branched to from within it.
- void getExitBlocks(Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const;
+ void getExitBlocks(CycleRef C, SmallVectorImpl<BlockT *> &TmpStorage) const;
/// Return all blocks of \p C that have a successor outside of \p C.
- void getExitingBlocks(Cycle C, SmallVectorImpl<BlockT *> &TmpStorage) const;
+ void getExitingBlocks(CycleRef C,
+ SmallVectorImpl<BlockT *> &TmpStorage) const;
/// Return the preheader block for \p C. Pre-header is well-defined for
/// reducible cycle in docs/LoopTerminology.md as: the only one entering
/// block and its only edge is to the entry block. Return null for
/// irreducible cycles.
- BlockT *getCyclePreheader(Cycle C) const;
+ BlockT *getCyclePreheader(CycleRef C) const;
/// If \p C has exactly one entry with exactly one predecessor, return it,
/// otherwise return nullptr.
- BlockT *getCyclePredecessor(Cycle C) const;
+ BlockT *getCyclePredecessor(CycleRef C) const;
/// Verify that \p C is actually a well-formed cycle in the CFG.
- void verifyCycle(Cycle C) const;
+ void verifyCycle(CycleRef C) const;
/// Verify the parent-child relations of \p C.
///
/// Note that this does \em not check that \p C is really a cycle in the CFG.
- void verifyCycleNest(Cycle C) const;
+ void verifyCycleNest(CycleRef C) const;
/// Assumes that \p C is the innermost cycle containing \p Block.
/// \p Block will be appended to \p C and all of its parent cycles.
/// \p Block will be added to BlockMap with \p C.
- void addBlockToCycle(BlockT *Block, Cycle C);
+ void addBlockToCycle(BlockT *Block, CycleRef C);
/// Methods for debug and self-test.
//@{
@@ -369,7 +368,7 @@ template <typename ContextT> class GenericCycleInfo {
void verify() const;
void print(raw_ostream &Out) const;
void dump() const { print(dbgs()); }
- Printable print(Cycle C) const;
+ Printable print(CycleRef C) const;
//@}
/// Iteration over top-level cycles.
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 16279bd5bf061..34ea33eced6cf 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -94,7 +94,7 @@ template <typename ContextT> class ModifiedPostOrder {
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using Cycle = typename CycleInfoT::Cycle;
+ using CycleRef = typename CycleInfoT::CycleRef;
using const_iterator = typename std::vector<BlockT *>::const_iterator;
ModifiedPostOrder(const ContextT &C) : Context(C) {}
@@ -132,11 +132,11 @@ template <typename ContextT> class ModifiedPostOrder {
SmallPtrSet<const BlockT *, 32> ReducibleCycleHeaders;
const ContextT &Context;
- void computeCyclePO(const CycleInfoT &CI, Cycle C,
+ void computeCyclePO(const CycleInfoT &CI, CycleRef C,
SmallPtrSetImpl<const BlockT *> &Finalized);
void computeStackPO(SmallVectorImpl<const BlockT *> &Stack,
- const CycleInfoT &CI, Cycle C,
+ const CycleInfoT &CI, CycleRef C,
SmallPtrSetImpl<const BlockT *> &Finalized);
};
@@ -269,7 +269,7 @@ template <typename ContextT> class GenericSyncDependenceAnalysis {
using InstructionT = typename ContextT::InstructionT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using Cycle = typename CycleInfoT::Cycle;
+ using CycleRef = typename CycleInfoT::CycleRef;
using ConstBlockSet = SmallPtrSet<const BlockT *, 4>;
using ModifiedPO = ModifiedPostOrder<ContextT>;
@@ -339,7 +339,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using Cycle = typename CycleInfoT::Cycle;
+ using CycleRef = typename CycleInfoT::CycleRef;
using SyncDependenceAnalysisT = GenericSyncDependenceAnalysis<ContextT>;
using DivergenceDescriptorT =
@@ -347,7 +347,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
using BlockLabelMapT = typename SyncDependenceAnalysisT::BlockLabelMap;
using TemporalDivergenceTuple =
- std::tuple<ConstValueRefT, InstructionT *, Cycle>;
+ std::tuple<ConstValueRefT, InstructionT *, CycleRef>;
GenericUniformityAnalysisImpl(const DominatorTreeT &DT, const CycleInfoT &CI,
const TargetTransformInfo *TTI)
@@ -420,7 +420,7 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
SmallVector<TemporalDivergenceTuple, 8> TemporalDivergenceList;
- void recordTemporalDivergence(ConstValueRefT, const InstructionT *, Cycle);
+ void recordTemporalDivergence(ConstValueRefT, const InstructionT *, CycleRef);
/// Check if an instruction with Custom uniformity can be proven uniform
/// based on its operands. This queries the target-specific callback.
@@ -462,13 +462,13 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
const DominatorTreeT &DT;
// Recognized cycles with divergent exits.
- SmallSetVector<Cycle, 8> DivergentExitCycles;
+ SmallSetVector<CycleRef, 8> DivergentExitCycles;
// Cycles assumed to be divergent.
//
// We don't use a set here because every insertion needs an explicit
// traversal of all existing members.
- SmallVector<Cycle> AssumedDivergent;
+ SmallVector<CycleRef> AssumedDivergent;
// The SDA links divergent branches to divergent control-flow joins.
SyncDependenceAnalysisT SDA;
@@ -487,19 +487,19 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
/// \brief Identify all Instructions that become divergent because \p DivExit
/// is a divergent cycle exit of \p DivCycle. Mark those instructions as
/// divergent and push them on the worklist.
- void propagateCycleExitDivergence(const BlockT &DivExit, Cycle DivCycle);
+ void propagateCycleExitDivergence(const BlockT &DivExit, CycleRef DivCycle);
/// Mark as divergent all external uses of values defined in \p DefCycle.
- void analyzeCycleExitDivergence(Cycle DefCycle);
+ void analyzeCycleExitDivergence(CycleRef DefCycle);
/// \brief Mark as divergent all uses of \p I that are outside \p DefCycle.
- void propagateTemporalDivergence(const InstructionT &I, Cycle DefCycle);
+ void propagateTemporalDivergence(const InstructionT &I, CycleRef DefCycle);
/// \brief Push all users of \p Val (in the region) to the worklist.
void pushUsers(const InstructionT &I);
void pushUsers(ConstValueRefT V);
- bool usesValueFromCycle(const InstructionT &I, Cycle DefCycle) const;
+ bool usesValueFromCycle(const InstructionT &I, CycleRef DefCycle) const;
/// \brief Whether \p Def is divergent when read in \p ObservingBlock.
bool isTemporalDivergent(const BlockT &ObservingBlock,
@@ -520,7 +520,7 @@ template <typename ContextT> class DivergencePropagator {
using ValueRefT = typename ContextT::ValueRefT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using Cycle = typename CycleInfoT::Cycle;
+ using CycleRef = typename CycleInfoT::CycleRef;
using ModifiedPO = ModifiedPostOrder<ContextT>;
using SyncDependenceAnalysisT = GenericSyncDependenceAnalysis<ContextT>;
@@ -634,17 +634,17 @@ template <typename ContextT> class DivergencePropagator {
<< Context.print(&DivTermBlock) << "\n");
int DivTermIdx = CyclePOT.getIndex(&DivTermBlock);
- Cycle DivTermCycle = CI.getCycle(&DivTermBlock);
+ CycleRef DivTermCycle = CI.getCycle(&DivTermBlock);
// Locate the largest ancestor cycle that is not reducible and does not
// contain a reducible ancestor. This is done with a lambda that is defined
// and invoked in the same statement.
- Cycle IrreducibleAncestor = [this](Cycle C) -> Cycle {
+ CycleRef IrreducibleAncestor = [this](CycleRef C) -> CycleRef {
if (!C)
- return Cycle();
+ return CycleRef();
if (CI.isReducible(C))
- return Cycle();
- while (Cycle P = CI.getParentCycle(C)) {
+ return CycleRef();
+ while (CycleRef P = CI.getParentCycle(C)) {
if (CI.isReducible(P))
return C;
C = P;
@@ -715,7 +715,7 @@ template <typename ContextT> class DivergencePropagator {
LLVM_DEBUG(dbgs() << "Check for reducible cycle: " << Context.print(Block)
<< '\n');
if (CyclePOT.isReducibleCycleHeader(Block)) {
- Cycle BlockCycle = CI.getCycle(Block);
+ CycleRef BlockCycle = CI.getCycle(Block);
LLVM_DEBUG(dbgs() << CI.print(BlockCycle) << '\n');
SmallVector<BlockT *, 4> BlockCycleExits;
CI.getExitBlocks(BlockCycle, BlockCycleExits);
@@ -737,7 +737,7 @@ template <typename ContextT> class DivergencePropagator {
// Check every cycle containing DivTermBlock for exit divergence.
// A cycle has exit divergence if the label of an exit block does
// not match the label of its header.
- for (Cycle C = CI.getCycle(&DivTermBlock); C; C = CI.getParentCycle(C)) {
+ for (auto C = CI.getCycle(&DivTermBlock); C; C = CI.getParentCycle(C)) {
if (CI.isReducible(C)) {
// The exit divergence of a reducible cycle is recorded while
// propagating labels.
@@ -873,7 +873,7 @@ void GenericUniformityAnalysisImpl<ContextT>::addCustomUniformityCandidate(
// need to be propagated as divergent at their use outside the cycle.
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
- Cycle DefCycle) {
+ CycleRef DefCycle) {
SmallVector<BlockT *> Exits;
CI.getExitBlocks(DefCycle, Exits);
for (auto *Exit : Exits) {
@@ -896,12 +896,12 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeCycleExitDivergence(
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::propagateCycleExitDivergence(
- const BlockT &DivExit, Cycle InnerDivCycle) {
+ const BlockT &DivExit, CycleRef InnerDivCycle) {
LLVM_DEBUG(dbgs() << "\tpropCycleExitDiv " << Context.print(&DivExit)
<< "\n");
- Cycle DivCycle = InnerDivCycle;
- Cycle OuterDivCycle = DivCycle;
- Cycle ExitLevelCycle = CI.getCycle(&DivExit);
+ CycleRef DivCycle = InnerDivCycle;
+ CycleRef OuterDivCycle = DivCycle;
+ CycleRef ExitLevelCycle = CI.getCycle(&DivExit);
const unsigned CycleExitDepth =
ExitLevelCycle ? CI.getDepth(ExitLevelCycle) : 0;
@@ -920,7 +920,7 @@ void GenericUniformityAnalysisImpl<ContextT>::propagateCycleExitDivergence(
// Exit divergence does not matter if the cycle itself is assumed to
// be divergent.
- for (Cycle C : AssumedDivergent) {
+ for (auto C : AssumedDivergent) {
if (CI.contains(C, OuterDivCycle))
return;
}
@@ -966,10 +966,11 @@ void GenericUniformityAnalysisImpl<ContextT>::taintAndPushPhiNodes(
/// Add \p Candidate to \p Cycles if it is not already contained in \p Cycles.
///
/// \return true iff \p Candidate was added to \p Cycles.
-template <typename CycleInfoT, typename CycleT>
-bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleT> &Cycles,
- CycleT Candidate) {
- if (llvm::any_of(Cycles, [&](CycleT C) { return CI.contains(C, Candidate); }))
+template <typename CycleInfoT, typename CycleRef>
+bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleRef> &Cycles,
+ CycleRef Candidate) {
+ if (llvm::any_of(Cycles,
+ [&](CycleRef C) { return CI.contains(C, Candidate); }))
return false;
Cycles.push_back(Candidate);
return true;
@@ -980,17 +981,17 @@ bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleT> &Cycles,
/// If two paths that diverged outside an irreducible cycle join
/// inside that cycle, then that whole cycle is assumed to be
/// divergent. This does not apply if the cycle is reducible.
-template <typename CycleInfoT, typename CycleT, typename BlockT>
-CycleT getExtDivCycle(const CycleInfoT &CI, CycleT Cycle,
- const BlockT *DivTermBlock, const BlockT *JoinBlock) {
+template <typename CycleInfoT, typename CycleRef, typename BlockT>
+CycleRef getExtDivCycle(const CycleInfoT &CI, CycleRef Cycle,
+ const BlockT *DivTermBlock, const BlockT *JoinBlock) {
assert(Cycle);
assert(CI.contains(Cycle, JoinBlock));
if (CI.contains(Cycle, DivTermBlock))
- return CycleT();
+ return CycleRef();
- CycleT OriginalCycle = Cycle;
- CycleT Parent = CI.getParentCycle(Cycle);
+ CycleRef OriginalCycle = Cycle;
+ CycleRef Parent = CI.getParentCycle(Cycle);
while (Parent && !CI.contains(Parent, DivTermBlock)) {
Cycle = Parent;
Parent = CI.getParentCycle(Cycle);
@@ -1004,7 +1005,7 @@ CycleT getExtDivCycle(const CycleInfoT &CI, CycleT Cycle,
if (CI.isReducible(Cycle)) {
assert(CI.getHeader(Cycle) == JoinBlock);
- return CycleT();
+ return CycleRef();
}
LLVM_DEBUG(dbgs() << "cycle made divergent by external branch\n");
@@ -1015,16 +1016,16 @@ CycleT getExtDivCycle(const CycleInfoT &CI, CycleT Cycle,
///
/// This checks the "diverged entry" criterion defined in the
/// docs/ConvergenceAnalysis.html.
-template <typename ContextT, typename CycleInfoT, typename CycleT,
+template <typename ContextT, typename CycleInfoT, typename CycleRef,
typename BlockT, typename DominatorTreeT>
-CycleT getIntDivCycle(const CycleInfoT &CI, CycleT Cycle,
- const BlockT *DivTermBlock, const BlockT *JoinBlock,
- const DominatorTreeT &DT, ContextT &Context) {
+CycleRef getIntDivCycle(const CycleInfoT &CI, CycleRef Cycle,
+ const BlockT *DivTermBlock, const BlockT *JoinBlock,
+ const DominatorTreeT &DT, ContextT &Context) {
LLVM_DEBUG(dbgs() << "examine join " << Context.print(JoinBlock)
<< " for internal branch " << Context.print(DivTermBlock)
<< "\n");
if (DT.properlyDominates(DivTermBlock, JoinBlock))
- return CycleT();
+ return CycleRef();
// Find the smallest common cycle, if one exists.
assert(Cycle && CI.contains(Cycle, JoinBlock));
@@ -1032,15 +1033,15 @@ CycleT getIntDivCycle(const CycleInfoT &CI, CycleT Cycle,
Cycle = CI.getParentCycle(Cycle);
}
if (!Cycle || CI.isReducible(Cycle))
- return CycleT();
+ return CycleRef();
if (DT.properlyDominates(CI.getHeader(Cycle), JoinBlock))
- return CycleT();
+ return CycleRef();
LLVM_DEBUG(dbgs() << " header " << Context.print(CI.getHeader(Cycle))
<< " does not dominate join\n");
- CycleT Parent = CI.getParentCycle(Cycle);
+ CycleRef Parent = CI.getParentCycle(Cycle);
while (Parent && !DT.properlyDominates(CI.getHeader(Parent), JoinBlock)) {
LLVM_DEBUG(dbgs() << " header " << Context.print(CI.getHeader(Parent))
<< " does not dominate join\n");
@@ -1052,21 +1053,22 @@ CycleT getIntDivCycle(const CycleInfoT &CI, CycleT Cycle,
return Cycle;
}
-template <typename ContextT, typename CycleInfoT, typename CycleT,
+template <typename ContextT, typename CycleInfoT, typename CycleRef,
typename BlockT, typename DominatorTreeT>
-CycleT getOutermostDivergentCycle(const CycleInfoT &CI, CycleT Cycle,
- const BlockT *DivTermBlock,
- const BlockT *JoinBlock,
- const DominatorTreeT &DT, ContextT &Context) {
+CycleRef
+getOutermostDivergentCycle(const CycleInfoT &CI, CycleRef Cycle,
+ const BlockT *DivTermBlock, const BlockT *JoinBlock,
+ const DominatorTreeT &DT, ContextT &Context) {
if (!Cycle)
- return CycleT();
+ return CycleRef();
// First try to expand Cycle to the largest that contains JoinBlock
// but not DivTermBlock.
- CycleT Ext = getExtDivCycle(CI, Cycle, DivTermBlock, JoinBlock);
+ CycleRef Ext = getExtDivCycle(CI, Cycle, DivTermBlock, JoinBlock);
// Continue expanding to the largest cycle that contains both.
- CycleT Int = getIntDivCycle(CI, Cycle, DivTermBlock, JoinBlock, DT, Context);
+ CycleRef Int =
+ getIntDivCycle(CI, Cycle, DivTermBlock, JoinBlock, DT, Context);
if (Int)
return Int;
@@ -1077,7 +1079,7 @@ template <typename ContextT>
bool GenericUniformityAnalysisImpl<ContextT>::isTemporalDivergent(
const BlockT &ObservingBlock, const InstructionT &Def) const {
const BlockT *DefBlock = Def.getParent();
- for (Cycle C = CI.getCycle(DefBlock); C && !CI.contains(C, &ObservingBlock);
+ for (auto C = CI.getCycle(DefBlock); C && !CI.contains(C, &ObservingBlock);
C = CI.getParentCycle(C)) {
if (DivergentExitCycles.contains(C)) {
return true;
@@ -1099,15 +1101,15 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
return;
const auto &DivDesc = SDA.getJoinBlocks(DivTermBlock);
- SmallVector<Cycle> DivCycles;
+ SmallVector<CycleRef> DivCycles;
// Iterate over all blocks now reachable by a disjoint path join
for (const auto *JoinBlock : DivDesc.JoinDivBlocks) {
- Cycle C = CI.getCycle(JoinBlock);
+ CycleRef C = CI.getCycle(JoinBlock);
LLVM_DEBUG(dbgs() << "visiting join block " << Context.print(JoinBlock)
<< "\n");
- if (Cycle Outermost = getOutermostDivergentCycle(CI, C, DivTermBlock,
- JoinBlock, DT, Context)) {
+ if (CycleRef Outermost = getOutermostDivergentCycle(
+ CI, C, DivTermBlock, JoinBlock, DT, Context)) {
LLVM_DEBUG(dbgs() << "found divergent cycle\n");
DivCycles.push_back(Outermost);
continue;
@@ -1117,7 +1119,7 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
// Sort by order of decreasing depth. This allows later cycles to be skipped
// because they are already contained in earlier ones.
- llvm::sort(DivCycles, [this](Cycle A, Cycle B) {
+ llvm::sort(DivCycles, [this](CycleRef A, CycleRef B) {
return CI.getDepth(A) > CI.getDepth(B);
});
@@ -1126,7 +1128,7 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
// the DFS chosen. Conservatively, all values produced in such a
// cycle are assumed divergent. "Cycle invariant" values may be
// assumed uniform, but that requires further analysis.
- for (Cycle C : DivCycles) {
+ for (auto C : DivCycles) {
if (!insertIfNotContained(CI, AssumedDivergent, C))
continue;
LLVM_DEBUG(dbgs() << "process divergent cycle\n");
@@ -1135,7 +1137,7 @@ void GenericUniformityAnalysisImpl<ContextT>::analyzeControlDivergence(
}
}
- Cycle BranchCycle = CI.getCycle(DivTermBlock);
+ CycleRef BranchCycle = CI.getCycle(DivTermBlock);
assert(DivDesc.CycleDivBlocks.empty() || BranchCycle);
for (const auto *DivExitBlock : DivDesc.CycleDivBlocks) {
propagateCycleExitDivergence(*DivExitBlock, BranchCycle);
@@ -1167,7 +1169,7 @@ void GenericUniformityAnalysisImpl<ContextT>::compute() {
template <typename ContextT>
void GenericUniformityAnalysisImpl<ContextT>::recordTemporalDivergence(
- ConstValueRefT Val, const InstructionT *User, Cycle C) {
+ ConstValueRefT Val, const InstructionT *User, CycleRef C) {
TemporalDivergenceList.emplace_back(Val, const_cast<InstructionT *>(User), C);
}
@@ -1205,7 +1207,7 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
if (!AssumedDivergent.empty()) {
FoundDivergence = true;
OS << "CYCLES ASSUMED DIVERGENT:\n";
- for (Cycle C : AssumedDivergent) {
+ for (auto C : AssumedDivergent) {
OS << " " << CI.print(C) << '\n';
}
}
@@ -1213,7 +1215,7 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
if (!DivergentExitCycles.empty()) {
FoundDivergence = true;
OS << "CYCLES WITH DIVERGENT EXIT:\n";
- for (Cycle C : DivergentExitCycles) {
+ for (auto C : DivergentExitCycles) {
OS << " " << CI.print(C) << '\n';
}
}
@@ -1323,7 +1325,7 @@ void GenericUniformityInfo<ContextT>::print(raw_ostream &Out) const {
template <typename ContextT>
void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
- SmallVectorImpl<const BlockT *> &Stack, const CycleInfoT &CI, Cycle C,
+ SmallVectorImpl<const BlockT *> &Stack, const CycleInfoT &CI, CycleRef C,
SmallPtrSetImpl<const BlockT *> &Finalized) {
LLVM_DEBUG(dbgs() << "inside computeStackPO\n");
while (!Stack.empty()) {
@@ -1334,7 +1336,7 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
}
LLVM_DEBUG(dbgs() << " visiting " << CI.getSSAContext().print(NextBB)
<< "\n");
- Cycle NestedCycle = CI.getCycle(NextBB);
+ CycleRef NestedCycle = CI.getCycle(NextBB);
if (C != NestedCycle &&
(!C || (NestedCycle && CI.contains(C, NestedCycle)))) {
LLVM_DEBUG(dbgs() << " found a cycle\n");
@@ -1393,7 +1395,8 @@ void llvm::ModifiedPostOrder<ContextT>::computeStackPO(
template <typename ContextT>
void ModifiedPostOrder<ContextT>::computeCyclePO(
- const CycleInfoT &CI, Cycle C, SmallPtrSetImpl<const BlockT *> &Finalized) {
+ const CycleInfoT &CI, CycleRef C,
+ SmallPtrSetImpl<const BlockT *> &Finalized) {
LLVM_DEBUG(dbgs() << "inside computeCyclePO\n");
SmallVector<const BlockT *> Stack;
auto *CycleHeader = CI.getHeader(C);
@@ -1437,7 +1440,7 @@ void llvm::ModifiedPostOrder<ContextT>::compute(const CycleInfoT &CI) {
auto *F = CI.getFunction();
Stack.reserve(24); // FIXME made-up number
Stack.push_back(&F->front());
- computeStackPO(Stack, CI, Cycle(), Finalized);
+ computeStackPO(Stack, CI, CycleRef(), Finalized);
}
} // namespace llvm
diff --git a/llvm/include/llvm/ADT/GenericUniformityInfo.h b/llvm/include/llvm/ADT/GenericUniformityInfo.h
index 88d9377f4c3b4..94d207c2f8cd2 100644
--- a/llvm/include/llvm/ADT/GenericUniformityInfo.h
+++ b/llvm/include/llvm/ADT/GenericUniformityInfo.h
@@ -38,10 +38,10 @@ template <typename ContextT> class GenericUniformityInfo {
using ThisT = GenericUniformityInfo<ContextT>;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using Cycle = typename CycleInfoT::Cycle;
+ using CycleRef = typename CycleInfoT::CycleRef;
using TemporalDivergenceTuple =
- std::tuple<ConstValueRefT, InstructionT *, Cycle>;
+ std::tuple<ConstValueRefT, InstructionT *, CycleRef>;
GenericUniformityInfo(const DominatorTreeT &DT, const CycleInfoT &CI,
const TargetTransformInfo *TTI = nullptr);
diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
index e7f7b8f921beb..5b5370681309d 100644
--- a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
+++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
@@ -23,7 +23,6 @@
namespace llvm {
class MachineCycleInfo : public GenericCycleInfo<MachineSSAContext> {};
-using MachineCycle = MachineCycleInfo::Cycle;
/// Legacy analysis pass which computes a \ref MachineCycleInfo.
class LLVM_ABI MachineCycleInfoWrapperPass : public MachineFunctionPass {
@@ -47,7 +46,7 @@ class LLVM_ABI MachineCycleInfoWrapperPass : public MachineFunctionPass {
// TODO: add this function to the GenericCycleInfo template after implementing
// the IR version.
LLVM_ABI bool isCycleInvariant(const MachineCycleInfo &CI,
- const MachineCycle &Cycle, MachineInstr &I);
+ const CycleRef &Cycle, MachineInstr &I);
class MachineCycleAnalysis : public AnalysisInfoMixin<MachineCycleAnalysis> {
friend AnalysisInfoMixin<MachineCycleAnalysis>;
diff --git a/llvm/include/llvm/IR/CycleInfo.h b/llvm/include/llvm/IR/CycleInfo.h
index 20f9acd03dc1a..4e11c2cc16ae6 100644
--- a/llvm/include/llvm/IR/CycleInfo.h
+++ b/llvm/include/llvm/IR/CycleInfo.h
@@ -23,8 +23,6 @@ namespace llvm {
// Use class instead of using to allow forward declarations.
class CycleInfo : public GenericCycleInfo<SSAContext> {};
-using Cycle = CycleInfo::Cycle;
-
} // namespace llvm
#endif // LLVM_IR_CYCLEINFO_H
diff --git a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
index 0decda51915ae..8d38c38c70f6f 100644
--- a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
+++ b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
@@ -132,7 +132,7 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
const auto &F = *Context.getFunction();
DenseMap<const BlockT *, SmallVector<const InstructionT *, 8>> LiveTokenMap;
- DenseMap<Cycle, const InstructionT *> CycleHearts;
+ DenseMap<CycleRef, const InstructionT *> CycleHearts;
// Just like the DominatorTree, compute the CycleInfo locally so that we
// can run the verifier outside of a pass manager and we don't rely on
@@ -153,7 +153,7 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
// Check static rules about cycles.
auto *BB = User->getParent();
- Cycle BBCycle = CI.getCycle(BB);
+ CycleRef BBCycle = CI.getCycle(BB);
if (!BBCycle)
return;
@@ -170,7 +170,7 @@ void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
{Context.print(User), CI.print(BBCycle)});
while (true) {
- Cycle Parent = CI.getParentCycle(BBCycle);
+ CycleRef Parent = CI.getParentCycle(BBCycle);
if (!Parent || CI.contains(Parent, DefBB))
break;
BBCycle = Parent;
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index a8fb10b0c3637..6edd5a1909abc 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -178,10 +178,10 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
}
}
- DenseSet<Cycle> CyclesWithHoles;
+ DenseSet<CycleRef> CyclesWithHoles;
if (CI && ExclusionSet) {
for (auto *BB : *ExclusionSet) {
- if (Cycle C = CI->getTopLevelParentCycle(BB))
+ if (CycleRef C = CI->getTopLevelParentCycle(BB))
CyclesWithHoles.insert(C);
}
}
@@ -194,10 +194,10 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
}
}
- DenseSet<Cycle> StopCycles;
+ DenseSet<CycleRef> StopCycles;
if (CI) {
for (auto *StopSetBB : StopSet) {
- if (Cycle C = CI->getTopLevelParentCycle(StopSetBB))
+ if (CycleRef C = CI->getTopLevelParentCycle(StopSetBB))
StopCycles.insert(C);
}
}
@@ -232,12 +232,12 @@ static bool isReachableImpl(SmallVectorImpl<BasicBlock *> &Worklist,
return true;
}
- Cycle OuterC;
+ CycleRef OuterC;
if (CI) {
OuterC = CI->getTopLevelParentCycle(BB);
if (OuterC) {
if (CyclesWithHoles.count(OuterC))
- OuterC = Cycle();
+ OuterC = CycleRef();
else if (StopCycles.contains(OuterC))
return true;
} else {
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index 7a2c2139f69ce..c07119d700285 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -109,7 +109,7 @@ template <> void llvm::GenericUniformityAnalysisImpl<SSAContext>::initialize() {
template <>
bool llvm::GenericUniformityAnalysisImpl<SSAContext>::usesValueFromCycle(
- const Instruction &I, Cycle DefCycle) const {
+ const Instruction &I, CycleRef DefCycle) const {
assert(!isAlwaysUniform(I));
for (const Use &U : I.operands()) {
if (auto *I = dyn_cast<Instruction>(&U)) {
@@ -123,7 +123,7 @@ bool llvm::GenericUniformityAnalysisImpl<SSAContext>::usesValueFromCycle(
template <>
void llvm::GenericUniformityAnalysisImpl<
SSAContext>::propagateTemporalDivergence(const Instruction &I,
- Cycle DefCycle) {
+ CycleRef DefCycle) {
for (auto *User : I.users()) {
auto *UserInstr = cast<Instruction>(User);
if (CI.contains(DefCycle, UserInstr->getParent()))
diff --git a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
index adefedc65ca55..ae7fd3213bfae 100644
--- a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
@@ -116,8 +116,8 @@ MachineCycleInfoPrinterPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
}
-bool llvm::isCycleInvariant(const MachineCycleInfo &CI,
- const MachineCycle &Cycle, MachineInstr &I) {
+bool llvm::isCycleInvariant(const MachineCycleInfo &CI, const CycleRef &Cycle,
+ MachineInstr &I) {
MachineFunction *MF = I.getParent()->getParent();
MachineRegisterInfo *MRI = &MF->getRegInfo();
const TargetSubtargetInfo &ST = MF->getSubtarget();
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 77eae30e22398..aefdc76f0e0fa 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -258,11 +258,11 @@ class MachineSinking {
bool &BreakPHIEdge,
AllSuccsCache &AllSuccessors);
- void FindCycleSinkCandidates(MachineCycle Cycle, MachineBasicBlock *BB,
+ void FindCycleSinkCandidates(CycleRef Cycle, MachineBasicBlock *BB,
SmallVectorImpl<MachineInstr *> &Candidates);
bool
- aggressivelySinkIntoCycle(MachineCycle Cycle, MachineInstr &I,
+ aggressivelySinkIntoCycle(CycleRef Cycle, MachineInstr &I,
DenseMap<SinkItem, MachineInstr *> &SunkInstrs);
bool isProfitableToSinkTo(Register Reg, MachineInstr &MI,
@@ -720,7 +720,7 @@ static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI) {
}
void MachineSinking::FindCycleSinkCandidates(
- MachineCycle Cycle, MachineBasicBlock *BB,
+ CycleRef Cycle, MachineBasicBlock *BB,
SmallVectorImpl<MachineInstr *> &Candidates) {
for (auto &MI : *BB) {
LLVM_DEBUG(dbgs() << "CycleSink: Analysing candidate: " << MI);
@@ -883,7 +883,7 @@ bool MachineSinking::run(MachineFunction &MF) {
}
if (SinkInstsIntoCycle) {
- SmallVector<MachineCycle, 8> Cycles(CI->toplevel_cycles());
+ SmallVector<CycleRef, 8> Cycles(CI->toplevel_cycles());
SchedModel.init(STI);
bool HasHighPressure;
@@ -894,7 +894,7 @@ bool MachineSinking::run(MachineFunction &MF) {
++Stage, SunkInstrs.clear()) {
HasHighPressure = false;
- for (MachineCycle Cycle : Cycles) {
+ for (auto Cycle : Cycles) {
MachineBasicBlock *Preheader = CI->getCyclePreheader(Cycle);
if (!Preheader) {
LLVM_DEBUG(dbgs() << "CycleSink: Can't find preheader\n");
@@ -1116,8 +1116,8 @@ bool MachineSinking::isLegalToBreakCriticalEdge(MachineInstr &MI,
if (!SplitEdges || FromBB == ToBB || !FromBB->isSuccessor(ToBB))
return false;
- MachineCycle FromCycle = CI->getCycle(FromBB);
- MachineCycle ToCycle = CI->getCycle(ToBB);
+ CycleRef FromCycle = CI->getCycle(FromBB);
+ CycleRef ToCycle = CI->getCycle(ToBB);
// Check for backedges of more "complex" cycles.
if (FromCycle == ToCycle && FromCycle &&
@@ -1304,7 +1304,7 @@ bool MachineSinking::isProfitableToSinkTo(Register Reg, MachineInstr &MI,
FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2, AllSuccessors);
- MachineCycle MCycle = CI->getCycle(MBB);
+ CycleRef MCycle = CI->getCycle(MBB);
// If the instruction is not inside a cycle, it is not profitable to sink MI
// to a post dominate block SuccToSinkTo.
@@ -1340,7 +1340,7 @@ bool MachineSinking::isProfitableToSinkTo(Register Reg, MachineInstr &MI,
MachineInstr *DefMI = MRI->getVRegDef(Reg);
if (!DefMI)
continue;
- MachineCycle Cycle = CI->getCycle(DefMI->getParent());
+ CycleRef Cycle = CI->getCycle(DefMI->getParent());
// DefMI is defined outside of cycle. There should be no live range
// impact for this operand. Defination outside of cycle means:
// 1: defination is outside of cycle.
@@ -1752,7 +1752,7 @@ bool MachineSinking::hasStoreBetween(MachineBasicBlock *From,
/// based on the amount of sinking, or the type of ops being sunk (so long as
/// they are safe to sink).
bool MachineSinking::aggressivelySinkIntoCycle(
- MachineCycle Cycle, MachineInstr &I,
+ CycleRef Cycle, MachineInstr &I,
DenseMap<SinkItem, MachineInstr *> &SunkInstrs) {
// TODO: support instructions with multiple defs
if (I.getNumDefs() > 1)
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index 49cf1a8920087..21c5cd75ee799 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -109,7 +109,7 @@ void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::pushUsers(
template <>
bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::usesValueFromCycle(
- const MachineInstr &I, MachineCycle DefCycle) const {
+ const MachineInstr &I, CycleRef DefCycle) const {
assert(!isAlwaysUniform(I));
for (auto &Op : I.operands()) {
if (!Op.isReg() || !Op.readsReg())
@@ -131,7 +131,7 @@ bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::usesValueFromCycle(
template <>
void llvm::GenericUniformityAnalysisImpl<
MachineSSAContext>::propagateTemporalDivergence(const MachineInstr &I,
- MachineCycle DefCycle) {
+ CycleRef DefCycle) {
const auto &RegInfo = F.getRegInfo();
for (auto &Op : I.all_defs()) {
if (!Op.getReg().isVirtual())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
index 55ca311886c92..1f8d2019293ff 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -240,14 +240,14 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
// In case of use outside muliple nested cycles or muliple uses we only need
// to merge lane mask across largest relevant cycle.
- SmallDenseMap<Register, std::pair<MachineCycle, Register>> LRCCache;
+ SmallDenseMap<Register, std::pair<CycleRef, Register>> LRCCache;
for (auto [Reg, UseInst, LRC] : MUI->getTemporalDivergenceList()) {
if (MRI->getType(Reg) != LLT::scalar(1))
continue;
auto [LRCCacheIter, RegNotCached] = LRCCache.try_emplace(Reg);
auto &CycleMergedMask = LRCCacheIter->getSecond();
- MachineCycle &CachedLRC = CycleMergedMask.first;
+ CycleRef &CachedLRC = CycleMergedMask.first;
if (RegNotCached || CInfo.contains(LRC, CachedLRC)) {
CachedLRC = LRC;
}
@@ -256,7 +256,7 @@ bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
for (auto &LRCCacheEntry : LRCCache) {
Register Reg = LRCCacheEntry.first;
auto &CycleMergedMask = LRCCacheEntry.getSecond();
- MachineCycle Cycle = CycleMergedMask.first;
+ CycleRef Cycle = CycleMergedMask.first;
Register MergedMask = MRI->createVirtualRegister(BoolS1);
SSAUpdater.Initialize(MergedMask);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c1aa477576e4f..167df06fabc77 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -228,11 +228,11 @@ bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
MachineInstr *SgprDef = MRI.getVRegDef(Op.getReg());
// SgprDef defined inside cycle
- MachineCycle FromCycle = CI->getCycle(SgprDef->getParent());
+ CycleRef FromCycle = CI->getCycle(SgprDef->getParent());
if (!FromCycle)
continue;
- MachineCycle ToCycle = CI->getCycle(SuccToSinkTo);
+ CycleRef ToCycle = CI->getCycle(SuccToSinkTo);
// Check if there is a FromCycle that contains SgprDef's basic block but
// does not contain SuccToSinkTo and also has divergent exit condition.
while (FromCycle && !(ToCycle && CI->contains(FromCycle, ToCycle))) {
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 4c5b341db0e83..8cc7714c6fa2e 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -67,7 +67,7 @@ class SILowerSGPRSpills {
MBBVector SaveBlocks;
MBBVector RestoreBlocks;
- MachineBasicBlock *getCycleDomBB(MachineCycle C);
+ MachineBasicBlock *getCycleDomBB(CycleRef C);
public:
SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
@@ -302,7 +302,7 @@ bool SILowerSGPRSpills::spillCalleeSavedRegs(
return false;
}
-MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(MachineCycle C) {
+MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(CycleRef C) {
// If the insertion point lands on a cycle entry, move it to a block that
// dominates all entries.
if (MCI->isReducible(C)) {
@@ -518,7 +518,7 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
LaneVGPRInsertPt IP = LaneVGPRDomInstr[Reg];
- if (MachineCycle C = MCI->getTopLevelParentCycle(IP.MBB)) {
+ if (CycleRef C = MCI->getTopLevelParentCycle(IP.MBB)) {
MachineBasicBlock *AdjMBB = getCycleDomBB(C);
IP = insertPt(AdjMBB, AdjMBB->getFirstTerminator());
}
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 38bf9df607b48..050bc32355253 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -214,11 +214,11 @@ ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
} // namespace llvm
static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
- bool HeaderOnly, Cycle *CPtr = nullptr) {
+ bool HeaderOnly, CycleRef *CPtr = nullptr) {
if (!CI)
return true;
auto *BB = I->getParent();
- Cycle C = CI->getCycle(BB);
+ CycleRef C = CI->getCycle(BB);
if (!C)
return false;
if (CPtr)
@@ -11421,7 +11421,7 @@ struct AAPotentialValuesFloating : AAPotentialValuesImpl {
A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
*PHI.getFunction());
- Cycle C;
+ CycleRef C;
bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 9ff8801181602..ed86bf8e68bbb 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1620,7 +1620,7 @@ bool DSEState::isGuaranteedLoopIndependent(const Instruction *Current,
// would also be valid but we currently disable that to limit compile time).
if (Current->getParent() == KillingDef->getParent())
return true;
- Cycle CurrentC = CI.getCycle(Current->getParent());
+ CycleRef CurrentC = CI.getCycle(Current->getParent());
if (CurrentC && CurrentC == CI.getCycle(KillingDef->getParent()))
return true;
// Otherwise check the memory location is invariant to any loops.
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index dd1c4723d9b4d..5d1faec67bddd 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -772,7 +772,8 @@ BasicBlock *llvm::SplitCallBrEdge(BasicBlock *CallBrBlock, BasicBlock *Succ,
CallBrTarget, Succ);
if (UpdatedLI)
*UpdatedLI = Updated;
- updateCycleLoopInfo<CycleInfo, Cycle>(CI, CallBrBlock, CallBrTarget, Succ);
+ updateCycleLoopInfo<CycleInfo, CycleRef>(CI, CallBrBlock, CallBrTarget,
+ Succ);
} else {
for (PHINode &PN : Succ->phis())
PN.removeIncomingValue(CallBrBlock, false);
diff --git a/llvm/lib/Transforms/Utils/FixIrreducible.cpp b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
index 63940b3777935..77e8cb838ca75 100644
--- a/llvm/lib/Transforms/Utils/FixIrreducible.cpp
+++ b/llvm/lib/Transforms/Utils/FixIrreducible.cpp
@@ -220,7 +220,7 @@ static void reconnectChildLoops(LoopInfo &LI, Loop *ParentLoop, Loop *NewLoop,
}
}
-static void updateLoopInfo(CycleInfo &CI, LoopInfo &LI, Cycle C,
+static void updateLoopInfo(CycleInfo &CI, LoopInfo &LI, CycleRef C,
ArrayRef<BasicBlock *> GuardBlocks) {
// The parent loop is a natural loop L mapped to the cycle header H as long as
// H is not also the header of L. In the latter case, L is destroyed and we
@@ -274,7 +274,7 @@ static void updateLoopInfo(CycleInfo &CI, LoopInfo &LI, Cycle C,
// Given a set of blocks and headers in an irreducible SCC, convert it into a
// natural loop. Also insert this new loop at its appropriate place in the
// hierarchy of loops.
-static bool fixIrreducible(Cycle C, CycleInfo &CI, DominatorTree &DT,
+static bool fixIrreducible(CycleRef C, CycleInfo &CI, DominatorTree &DT,
LoopInfo *LI) {
if (CI.isReducible(C))
return false;
@@ -416,7 +416,7 @@ static bool fixIrreducible(Cycle C, CycleInfo &CI, DominatorTree &DT,
CI.setSingleEntry(C, GuardBlocks[0]);
CI.verifyCycle(C);
- if (Cycle Parent = CI.getParentCycle(C))
+ if (CycleRef Parent = CI.getParentCycle(C))
CI.verifyCycle(Parent);
LLVM_DEBUG(dbgs() << "Finished one cycle:\n"; CI.print(dbgs()););
@@ -429,7 +429,7 @@ static bool FixIrreducibleImpl(Function &F, CycleInfo &CI, DominatorTree &DT,
<< F.getName() << "\n");
bool Changed = false;
- for (Cycle C : CI.cycles())
+ for (auto C : CI.cycles())
Changed |= fixIrreducible(C, CI, DT, LI);
if (!Changed)
>From 8e1d3f12a711e75523eb7c0719568e0616d541c9 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Thu, 16 Jul 2026 20:18:30 -0700
Subject: [PATCH 4/6] remove unneeded `using`
---
.../llvm/ADT/GenericConvergenceVerifier.h | 1 -
llvm/include/llvm/ADT/GenericCycleInfo.h | 19 +++++--------------
llvm/include/llvm/ADT/GenericUniformityImpl.h | 4 ----
llvm/include/llvm/ADT/GenericUniformityInfo.h | 1 -
4 files changed, 5 insertions(+), 20 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
index 3ce490d43acdc..bc4efe25eb05c 100644
--- a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
+++ b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h
@@ -28,7 +28,6 @@ template <typename ContextT> class GenericConvergenceVerifier {
using InstructionT = typename ContextT::InstructionT;
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleRef = typename CycleInfoT::CycleRef;
void initialize(raw_ostream *OS,
function_ref<void(const Twine &Message)> FailureCB,
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 53f744f082038..aaff2027d0ae5 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -28,7 +28,7 @@
#ifndef LLVM_ADT_GENERICCYCLEINFO_H
#define LLVM_ADT_GENERICCYCLEINFO_H
-#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/GenericSSAContext.h"
#include "llvm/ADT/GraphTraits.h"
#include "llvm/ADT/SetVector.h"
@@ -93,14 +93,10 @@ template <typename ContextT> class GenericCycle {
GenericCycle() = default;
};
-/// Opaque handle to a cycle within a GenericCycleInfo. Wraps the cycle's
-/// preorder index; a default-constructed handle is invalid ("no cycle"). All
-/// queries live on GenericCycleInfo, which resolves the handle to storage.
-///
-/// The handle is context-free: IR and machine cycles share one type, which
-/// keeps forward declarations simple. Handles remain valid as long as the
-/// cycle forest is not recomputed; addBlockToCycle() adds a block but never
-/// adds, removes, or reorders cycles, so it leaves every handle valid.
+/// Opaque handle to a cycle within a GenericCycleInfo that wraps the cycle's
+/// preorder index. Handles remain valid as long as the cycle forest is not
+/// recomputed; addBlockToCycle() adds a block but never adds, removes, or
+/// reorders cycles, so it leaves every handle valid.
class CycleRef {
static constexpr unsigned InvalidIndex = ~0u;
unsigned Index = InvalidIndex;
@@ -117,9 +113,6 @@ class CycleRef {
bool operator!=(CycleRef O) const { return Index != O.Index; }
};
-/// DenseMap tracks bucket occupancy with a separate bitmap rather than sentinel
-/// key values, so only hashing and equality are needed and the invalid handle
-/// is itself a legal key.
template <> struct DenseMapInfo<CycleRef> {
static unsigned getHashValue(CycleRef C) { return C.Index; }
static bool isEqual(CycleRef A, CycleRef B) { return A.Index == B.Index; }
@@ -131,8 +124,6 @@ template <typename ContextT> class GenericCycleInfo {
using BlockT = typename ContextT::BlockT;
/// The internal, by-value storage type for a cycle.
using CycleT = GenericCycle<ContextT>;
- /// The opaque handle by which consumers refer to a cycle.
- using CycleRef = ::llvm::CycleRef;
using FunctionT = typename ContextT::FunctionT;
template <typename> friend class GenericCycleInfoCompute;
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 34ea33eced6cf..99d3219194290 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -94,7 +94,6 @@ template <typename ContextT> class ModifiedPostOrder {
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleRef = typename CycleInfoT::CycleRef;
using const_iterator = typename std::vector<BlockT *>::const_iterator;
ModifiedPostOrder(const ContextT &C) : Context(C) {}
@@ -269,7 +268,6 @@ template <typename ContextT> class GenericSyncDependenceAnalysis {
using InstructionT = typename ContextT::InstructionT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleRef = typename CycleInfoT::CycleRef;
using ConstBlockSet = SmallPtrSet<const BlockT *, 4>;
using ModifiedPO = ModifiedPostOrder<ContextT>;
@@ -339,7 +337,6 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
using DominatorTreeT = typename ContextT::DominatorTreeT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleRef = typename CycleInfoT::CycleRef;
using SyncDependenceAnalysisT = GenericSyncDependenceAnalysis<ContextT>;
using DivergenceDescriptorT =
@@ -520,7 +517,6 @@ template <typename ContextT> class DivergencePropagator {
using ValueRefT = typename ContextT::ValueRefT;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleRef = typename CycleInfoT::CycleRef;
using ModifiedPO = ModifiedPostOrder<ContextT>;
using SyncDependenceAnalysisT = GenericSyncDependenceAnalysis<ContextT>;
diff --git a/llvm/include/llvm/ADT/GenericUniformityInfo.h b/llvm/include/llvm/ADT/GenericUniformityInfo.h
index 94d207c2f8cd2..594d6f8bd1f60 100644
--- a/llvm/include/llvm/ADT/GenericUniformityInfo.h
+++ b/llvm/include/llvm/ADT/GenericUniformityInfo.h
@@ -38,7 +38,6 @@ template <typename ContextT> class GenericUniformityInfo {
using ThisT = GenericUniformityInfo<ContextT>;
using CycleInfoT = GenericCycleInfo<ContextT>;
- using CycleRef = typename CycleInfoT::CycleRef;
using TemporalDivergenceTuple =
std::tuple<ConstValueRefT, InstructionT *, CycleRef>;
>From 6829d880ca1f729db0993ed071cecf7e1c84419c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Thu, 16 Jul 2026 21:04:18 -0700
Subject: [PATCH 5/6] simplify cycles()
---
llvm/include/llvm/ADT/GenericCycleInfo.h | 27 +++++-------------------
1 file changed, 5 insertions(+), 22 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index aaff2027d0ae5..d8bac633d905c 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -31,6 +31,8 @@
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/GenericSSAContext.h"
#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/iterator.h"
@@ -204,25 +206,6 @@ template <typename ContextT> class GenericCycleInfo {
}
};
- /// Sequential iteration over all cycles in forest preorder, yielding handles.
- struct const_cycle_iterator
- : iterator_facade_base<const_cycle_iterator, std::forward_iterator_tag,
- CycleRef, std::ptrdiff_t, CycleRef, CycleRef> {
- unsigned Index = 0;
-
- const_cycle_iterator() = default;
- explicit const_cycle_iterator(unsigned Index) : Index(Index) {}
-
- CycleRef operator*() const { return CycleRef(Index); }
- const_cycle_iterator &operator++() {
- ++Index;
- return *this;
- }
- bool operator==(const const_cycle_iterator &Other) const {
- return Index == Other.Index;
- }
- };
-
GenericCycleInfo() = default;
GenericCycleInfo(GenericCycleInfo &&) = default;
GenericCycleInfo &operator=(GenericCycleInfo &&) = default;
@@ -235,9 +218,9 @@ template <typename ContextT> class GenericCycleInfo {
const ContextT &getSSAContext() const { return Context; }
/// All cycles in forest preorder.
- iterator_range<const_cycle_iterator> cycles() const {
- return llvm::make_range(const_cycle_iterator(0),
- const_cycle_iterator(NumCycles));
+ auto cycles() const {
+ return map_range(seq(0u, NumCycles),
+ [](unsigned I) { return CycleRef(I); });
}
/// \brief Find the innermost cycle containing \p Block.
>From 46be81f78916d3bc8c563ae2028854be1d5861be Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Fri, 17 Jul 2026 00:09:05 -0700
Subject: [PATCH 6/6] Make GenericCycle nested inside GenericCycleInfo
---
llvm/include/llvm/ADT/GenericCycleInfo.h | 96 +++++++++----------
llvm/include/llvm/ADT/GenericUniformityImpl.h | 12 +--
.../llvm/CodeGen/MachineCycleAnalysis.h | 4 +-
llvm/lib/CodeGen/MachineCycleAnalysis.cpp | 3 +-
llvm/lib/IR/CycleInfo.cpp | 1 -
5 files changed, 52 insertions(+), 64 deletions(-)
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index d8bac633d905c..d827f963b8d79 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -45,56 +45,6 @@ namespace llvm {
template <typename ContextT> class GenericCycleInfo;
template <typename ContextT> class GenericCycleInfoCompute;
-/// A possibly irreducible generalization of a \ref Loop.
-///
-/// Cycles are stored by value in GenericCycleInfo::Cycles in preorder of the
-/// cycle forest: a cycle is immediately followed by its descendants. Child
-/// iteration is therefore pointer arithmetic over that array.
-template <typename ContextT> class GenericCycle {
-public:
- using BlockT = typename ContextT::BlockT;
- using FunctionT = typename ContextT::FunctionT;
- template <typename> friend class GenericCycleInfo;
- template <typename> friend class GenericCycleInfoCompute;
-
-private:
- /// The parent cycle. Is null for top-level cycles.
- GenericCycle *ParentCycle = nullptr;
-
- /// The entry block(s) of the cycle. The header is the only entry if
- /// this is a loop.
- SmallVector<BlockT *, 1> Entries;
-
- /// This cycle's blocks (its own and its nested cycles') occupy the half-open
- /// range [IdxBegin, IdxEnd) of GenericCycleInfo::BlockLayout. The
- /// ranges are nested like an Euler tour of the cycle tree, so containment is
- /// an interval test (see contains()).
- ///
- /// During construction (before the forest is flattened), IdxEnd accumulates
- /// the number of this cycle's own blocks (those whose innermost cycle is
- /// this one).
- unsigned IdxBegin = 0, IdxEnd = 0;
-
- /// Depth of the cycle in the tree: top-level cycles are at depth 1 and each
- /// nested cycle is one deeper (getCycleDepth() returns 0 for blocks outside
- /// any cycle). Sibling cycles share a depth.
- unsigned Depth = 0;
-
- /// Number of cycles nested inside this one: the subtree occupies
- /// [this, this + 1 + NumDescendants) of GenericCycleInfo::Cycles.
- unsigned NumDescendants = 0;
-
- void appendEntry(BlockT *Block) { Entries.push_back(Block); }
-
- GenericCycle(const GenericCycle &) = delete;
- GenericCycle &operator=(const GenericCycle &) = delete;
- GenericCycle(GenericCycle &&Rhs) = delete;
- GenericCycle &operator=(GenericCycle &&Rhs) = delete;
-
-public:
- GenericCycle() = default;
-};
-
/// Opaque handle to a cycle within a GenericCycleInfo that wraps the cycle's
/// preorder index. Handles remain valid as long as the cycle forest is not
/// recomputed; addBlockToCycle() adds a block but never adds, removes, or
@@ -116,7 +66,9 @@ class CycleRef {
};
template <> struct DenseMapInfo<CycleRef> {
- static unsigned getHashValue(CycleRef C) { return C.Index; }
+ static unsigned getHashValue(CycleRef C) {
+ return DenseMapInfo<unsigned>::getHashValue(C.Index);
+ }
static bool isEqual(CycleRef A, CycleRef B) { return A.Index == B.Index; }
};
@@ -124,12 +76,50 @@ template <> struct DenseMapInfo<CycleRef> {
template <typename ContextT> class GenericCycleInfo {
public:
using BlockT = typename ContextT::BlockT;
- /// The internal, by-value storage type for a cycle.
- using CycleT = GenericCycle<ContextT>;
using FunctionT = typename ContextT::FunctionT;
template <typename> friend class GenericCycleInfoCompute;
private:
+ /// Internal, data-only storage for a cycle. Consumers name a cycle by a
+ /// CycleRef handle and query it through GenericCycleInfo.
+ class Cycle {
+ public:
+ /// The parent cycle. Is null for top-level cycles.
+ Cycle *ParentCycle = nullptr;
+
+ /// The entry block(s) of the cycle. The header is the only entry if this
+ /// is a loop.
+ SmallVector<BlockT *, 1> Entries;
+
+ /// This cycle's blocks (its own and its nested cycles') occupy the
+ /// half-open range [IdxBegin, IdxEnd) of BlockLayout, nested like an Euler
+ /// tour of the cycle tree, so containment is an interval test (see
+ /// contains()).
+ ///
+ /// During construction (before the forest is flattened), IdxEnd accumulates
+ /// the number of this cycle's own blocks (those whose innermost cycle is
+ /// this one).
+ unsigned IdxBegin = 0, IdxEnd = 0;
+
+ /// Depth of the cycle in the tree: top-level cycles are at depth 1 and each
+ /// nested cycle is one deeper (getCycleDepth() returns 0 for blocks outside
+ /// any cycle). Sibling cycles share a depth.
+ unsigned Depth = 0;
+
+ /// Number of cycles nested inside this one: the subtree occupies
+ /// [this, this + 1 + NumDescendants) of Cycles.
+ unsigned NumDescendants = 0;
+
+ void appendEntry(BlockT *Block) { Entries.push_back(Block); }
+
+ Cycle() = default;
+ Cycle(const Cycle &) = delete;
+ Cycle &operator=(const Cycle &) = delete;
+ Cycle(Cycle &&) = delete;
+ Cycle &operator=(Cycle &&) = delete;
+ };
+ using CycleT = Cycle;
+
ContextT Context;
unsigned BlockNumberEpoch;
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index 99d3219194290..737efde1d5e88 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -962,7 +962,7 @@ void GenericUniformityAnalysisImpl<ContextT>::taintAndPushPhiNodes(
/// Add \p Candidate to \p Cycles if it is not already contained in \p Cycles.
///
/// \return true iff \p Candidate was added to \p Cycles.
-template <typename CycleInfoT, typename CycleRef>
+template <typename CycleInfoT>
bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleRef> &Cycles,
CycleRef Candidate) {
if (llvm::any_of(Cycles,
@@ -977,7 +977,7 @@ bool insertIfNotContained(const CycleInfoT &CI, SmallVector<CycleRef> &Cycles,
/// If two paths that diverged outside an irreducible cycle join
/// inside that cycle, then that whole cycle is assumed to be
/// divergent. This does not apply if the cycle is reducible.
-template <typename CycleInfoT, typename CycleRef, typename BlockT>
+template <typename CycleInfoT, typename BlockT>
CycleRef getExtDivCycle(const CycleInfoT &CI, CycleRef Cycle,
const BlockT *DivTermBlock, const BlockT *JoinBlock) {
assert(Cycle);
@@ -1012,8 +1012,8 @@ CycleRef getExtDivCycle(const CycleInfoT &CI, CycleRef Cycle,
///
/// This checks the "diverged entry" criterion defined in the
/// docs/ConvergenceAnalysis.html.
-template <typename ContextT, typename CycleInfoT, typename CycleRef,
- typename BlockT, typename DominatorTreeT>
+template <typename ContextT, typename CycleInfoT, typename BlockT,
+ typename DominatorTreeT>
CycleRef getIntDivCycle(const CycleInfoT &CI, CycleRef Cycle,
const BlockT *DivTermBlock, const BlockT *JoinBlock,
const DominatorTreeT &DT, ContextT &Context) {
@@ -1049,8 +1049,8 @@ CycleRef getIntDivCycle(const CycleInfoT &CI, CycleRef Cycle,
return Cycle;
}
-template <typename ContextT, typename CycleInfoT, typename CycleRef,
- typename BlockT, typename DominatorTreeT>
+template <typename ContextT, typename CycleInfoT, typename BlockT,
+ typename DominatorTreeT>
CycleRef
getOutermostDivergentCycle(const CycleInfoT &CI, CycleRef Cycle,
const BlockT *DivTermBlock, const BlockT *JoinBlock,
diff --git a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
index 5b5370681309d..341175a562fe0 100644
--- a/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
+++ b/llvm/include/llvm/CodeGen/MachineCycleAnalysis.h
@@ -45,8 +45,8 @@ class LLVM_ABI MachineCycleInfoWrapperPass : public MachineFunctionPass {
// TODO: add this function to the GenericCycleInfo template after implementing
// the IR version.
-LLVM_ABI bool isCycleInvariant(const MachineCycleInfo &CI,
- const CycleRef &Cycle, MachineInstr &I);
+LLVM_ABI bool isCycleInvariant(const MachineCycleInfo &CI, CycleRef Cycle,
+ MachineInstr &I);
class MachineCycleAnalysis : public AnalysisInfoMixin<MachineCycleAnalysis> {
friend AnalysisInfoMixin<MachineCycleAnalysis>;
diff --git a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
index ae7fd3213bfae..8118e99a1ed52 100644
--- a/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineCycleAnalysis.cpp
@@ -17,7 +17,6 @@
using namespace llvm;
template class llvm::GenericCycleInfo<llvm::MachineSSAContext>;
-template class llvm::GenericCycle<llvm::MachineSSAContext>;
char MachineCycleInfoWrapperPass::ID = 0;
@@ -116,7 +115,7 @@ MachineCycleInfoPrinterPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
}
-bool llvm::isCycleInvariant(const MachineCycleInfo &CI, const CycleRef &Cycle,
+bool llvm::isCycleInvariant(const MachineCycleInfo &CI, CycleRef Cycle,
MachineInstr &I) {
MachineFunction *MF = I.getParent()->getParent();
MachineRegisterInfo *MRI = &MF->getRegInfo();
diff --git a/llvm/lib/IR/CycleInfo.cpp b/llvm/lib/IR/CycleInfo.cpp
index a9b9129f24f09..5353a303cb155 100644
--- a/llvm/lib/IR/CycleInfo.cpp
+++ b/llvm/lib/IR/CycleInfo.cpp
@@ -13,4 +13,3 @@
using namespace llvm;
template class llvm::GenericCycleInfo<SSAContext>;
-template class llvm::GenericCycle<SSAContext>;
More information about the llvm-commits
mailing list