[llvm] [AMDGPU] Add islands to StructurizeCFG (PR #201675)
Robert Imschweiler via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 06:22:31 PDT 2026
https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/201675
>From db33161469f5619de468ec95f75d721fccbafd2d Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 4 Jun 2026 14:02:06 -0500
Subject: [PATCH 1/2] [AMDGPU] Add islands to StructurizeCFG
Handle an "island", i.e. a block whose terminator is not rewritable
(e.g., a callbr) in this context and considered immutable from this
pass's perspective.
The island is left untouched; its edges are split into forwarders that
converge at a new Flow block (ExitFlow). The callbr's runtime target
choice is recovered as a per-target i1 "sel" phi and re-dispatched by a
structured ladder of 2-way branches:
callbr
/ | \
fwd_0 fwd_1 fwd_2 forwarders (intermediate target blocks)
\ | /
ExitFlow (Flow_0) -- sel_0? --> real_0
| else
Flow_1 --------- sel_1? --> real_1
| else
real_2
or Flow_2 (if real_1 == real_2) ------> real_2 (shared last target)
---
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 392 ++++-
llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll | 4 +-
llvm/test/CodeGen/AMDGPU/callbr.ll | 40 +-
llvm/test/CodeGen/AMDGPU/infinite-loop.ll | 160 +-
.../si-annotate-nested-control-flows.ll | 89 +-
.../si-unify-exit-multiple-unreachables.ll | 129 +-
llvm/test/Transforms/StructurizeCFG/callbr.ll | 1396 +++++++++++++++--
7 files changed, 1943 insertions(+), 267 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 04ece92b74375..d1ada89d9d246 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -14,6 +14,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/RegionInfo.h"
#include "llvm/Analysis/RegionIterator.h"
@@ -28,6 +29,8 @@
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
@@ -59,6 +62,32 @@ using namespace llvm::PatternMatch;
// The name for newly created blocks.
const char FlowBlockName[] = "Flow";
+// True if BB contains nothing but an unconditional branch and has only one
+// predecessor.
+static bool isIntermediateTarget(const BasicBlock &BB) {
+ return BB.size() == 1 && isa<UncondBrInst>(BB.getTerminator()) &&
+ BB.hasNPredecessors(1);
+}
+
+/// True if BB's terminator must be preserved verbatim and therefore cannot
+/// serve as a rewritable flow tail (it can be neither killed nor have its
+/// successors rewritten). Currently only callbr.
+static bool isIsland(const BasicBlock &BB) {
+ return isa<CallBrInst>(BB.getTerminator());
+}
+
+/// True if BB is an unreachable target block: it is terminated by
+/// `unreachable`, or (after exit unification) it is marked by an
+/// @llvm.amdgcn.unreachable call.
+static bool isUnreachableTarget(const BasicBlock &BB) {
+ if (isa<UnreachableInst>(BB.getTerminator()))
+ return true;
+ return any_of(BB, [](const Instruction &I) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ return CB && CB->getIntrinsicID() == Intrinsic::amdgcn_unreachable;
+ });
+}
+
namespace {
static cl::opt<bool> ForceSkipUniformRegions(
@@ -227,6 +256,52 @@ class NearestCommonDominator {
bool resultIsRememberedBlock() { return ResultIsRemembered; }
};
+class StructurizeCFG; // forward declaration
+
+/// The state of the current flow tail, i.e. the node that is the current tail
+/// of our structurization chain:
+/// - None: region entry, no pending tail yet
+/// - Rewritable: node with a rewritable terminator
+/// - Closed: node with a non-rewritable terminator that cannot be used for
+/// flow stitching
+class FlowTail {
+ enum class TailState { None, Rewritable, Closed };
+
+ RegionNode *Node;
+ TailState State;
+
+public:
+ FlowTail() : Node(nullptr), State(TailState::None) {}
+ FlowTail(RegionNode *Node)
+ : Node(Node), State(Node ? TailState::Rewritable : TailState::None) {}
+
+ RegionNode &operator*() {
+ assert(isRewritable() && "FlowTail is not rewritable");
+ return *Node;
+ }
+
+ RegionNode *operator->() {
+ assert(isRewritable() && "FlowTail is not rewritable");
+ return Node;
+ }
+
+ /// Close the flow tail, marking the end of an island.
+ void close(RegionNode *Boundary) {
+ assert(Boundary && "closing requires a boundary node");
+ Node = Boundary;
+ State = TailState::Closed;
+ }
+
+ RegionNode *getClosedEntry() const {
+ assert(isClosed() && "FlowTail is not closed");
+ return Node;
+ }
+
+ bool isClosed() const { return State == TailState::Closed; }
+ bool isNone() const { return State == TailState::None; }
+ bool isRewritable() const { return State == TailState::Rewritable; }
+};
+
/// Transforms the control flow graph on one single entry/exit region
/// at a time.
///
@@ -302,7 +377,11 @@ class StructurizeCFG {
Val2BBMap HoistedValues;
- RegionNode *PrevNode;
+ FlowTail Tail;
+
+ // True if the region exit domination info needs to be fixed after the
+ // transform due to specific island requirements, see handleIsland.
+ bool NeedFixRegionExitDom = false;
void hoistZeroCostElseBlockPhiValues(BasicBlock *ElseBB, BasicBlock *ThenBB);
@@ -342,8 +421,9 @@ class StructurizeCFG {
DebugLoc killTerminator(BasicBlock *BB);
- void changeExit(RegionNode *Node, BasicBlock *NewExit,
- bool IncludeDominator);
+ void changeExit(FlowTail &Tail, BasicBlock *NewExit, bool IncludeDominator);
+
+ void checkClosedFlow();
BasicBlock *getNextFlow(BasicBlock *Dominator);
@@ -359,6 +439,8 @@ class StructurizeCFG {
void wireFlow(bool ExitUseAllowed, BasicBlock *LoopEnd);
+ void handleIsland(BasicBlock *CallBrBlock);
+
void handleLoops(bool ExitUseAllowed, BasicBlock *LoopEnd);
void createFlow();
@@ -561,9 +643,8 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) {
} else {
// Test for successors as back edge
- // TODO: support other terminators other than branches.
BasicBlock *BB = N->getNodeAs<BasicBlock>();
- if (isa<UncondBrInst, CondBrInst>(BB->getTerminator()))
+ if (isa<UncondBrInst, CondBrInst, CallBrInst>(BB->getTerminator()))
for (BasicBlock *Succ : successors(BB))
if (Visited.count(Succ))
Loops[Succ] = BB;
@@ -620,6 +701,14 @@ void StructurizeCFG::gatherPredicates(RegionNode *N) {
LPred[P] = buildCondition(CondBr, Idx, true);
}
}
+ // } else if (isIsland(*P)) {
+ // // Island edges have no SSA condition, so we treat them as an
+ // // unconditional branch.
+ // if (Visited.count(P))
+ // Pred[P] = {BoolTrue, std::nullopt};
+ // else
+ // LPred[P] = {BoolFalse, std::nullopt};
+ // }
} else {
// It's an exit from a sub region
while (R->getParent() != ParentRegion)
@@ -1037,10 +1126,10 @@ DebugLoc StructurizeCFG::killTerminator(BasicBlock *BB) {
}
/// Let node exit(s) point to NewExit
-void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
+void StructurizeCFG::changeExit(FlowTail &Tail, BasicBlock *NewExit,
bool IncludeDominator) {
- if (Node->isSubRegion()) {
- Region *SubRegion = Node->getNodeAs<Region>();
+ if (Tail->isSubRegion()) {
+ Region *SubRegion = Tail->getNodeAs<Region>();
BasicBlock *OldExit = SubRegion->getExit();
BasicBlock *Dominator = nullptr;
@@ -1071,7 +1160,7 @@ void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
// Update the region info
SubRegion->replaceExit(NewExit);
} else {
- BasicBlock *BB = Node->getNodeAs<BasicBlock>();
+ BasicBlock *BB = Tail->getNodeAs<BasicBlock>();
DebugLoc DL = killTerminator(BB);
UncondBrInst *Br = UncondBrInst::Create(NewExit, BB);
Br->setDebugLoc(DL);
@@ -1081,6 +1170,18 @@ void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
}
}
+/// A closed flow tail marks an island boundary that handleIsland already wired
+/// (the callbr itself, whose terminator is immutable, or one of its forwarders,
+/// already converged into the dispatch ladder; see wireFlow). Such a node must
+/// not be rewritten or extended, so before more flow is stitched onto the tail
+/// (needPrefix) swap in a fresh, rewritable Flow node to continue from.
+void StructurizeCFG::checkClosedFlow() {
+ if (!Tail.isClosed())
+ return;
+ BasicBlock *Flow = getNextFlow(ParentRegion->getEntry());
+ Tail = ParentRegion->getBBNode(Flow);
+}
+
/// Create a new flow node and update dominator tree and region info
BasicBlock *StructurizeCFG::getNextFlow(BasicBlock *Dominator) {
LLVMContext &Context = Func->getContext();
@@ -1097,9 +1198,10 @@ BasicBlock *StructurizeCFG::getNextFlow(BasicBlock *Dominator) {
/// Create a new or reuse the previous node as flow node. Returns a block and a
/// debug location to be used for new instructions in that block.
std::pair<BasicBlock *, DebugLoc> StructurizeCFG::needPrefix(bool NeedEmpty) {
- BasicBlock *Entry = PrevNode->getEntry();
+ checkClosedFlow();
+ BasicBlock *Entry = Tail->getEntry();
- if (!PrevNode->isSubRegion()) {
+ if (!Tail->isSubRegion()) {
DebugLoc DL = killTerminator(Entry);
if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
return {Entry, DL};
@@ -1109,8 +1211,8 @@ std::pair<BasicBlock *, DebugLoc> StructurizeCFG::needPrefix(bool NeedEmpty) {
BasicBlock *Flow = getNextFlow(Entry);
// and wire it up
- changeExit(PrevNode, Flow, true);
- PrevNode = ParentRegion->getBBNode(Flow);
+ changeExit(Tail, Flow, true);
+ Tail = ParentRegion->getBBNode(Flow);
return {Flow, DebugLoc()};
}
@@ -1128,8 +1230,7 @@ BasicBlock *StructurizeCFG::needPostfix(BasicBlock *Flow,
/// Set the previous node
void StructurizeCFG::setPrevNode(BasicBlock *BB) {
- PrevNode = ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB)
- : nullptr;
+ Tail = ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB) : nullptr;
}
/// Does BB dominate all the predicates of Node?
@@ -1146,14 +1247,15 @@ bool StructurizeCFG::isPredictableTrue(RegionNode *Node) {
bool Dominated = false;
// Regionentry is always true
- if (!PrevNode)
+ // If we're in a closed flow, we assume that anything might be called.
+ if (Tail.isNone() || Tail.isClosed())
return true;
for (auto [BB, PI] : Preds) {
if (PI.Pred != BoolTrue)
return false;
- if (!Dominated && DT->dominates(BB, PrevNode->getEntry()))
+ if (!Dominated && DT->dominates(BB, Tail->getEntry()))
Dominated = true;
}
@@ -1165,20 +1267,41 @@ bool StructurizeCFG::isPredictableTrue(RegionNode *Node) {
void StructurizeCFG::wireFlow(bool ExitUseAllowed,
BasicBlock *LoopEnd) {
RegionNode *Node = Order.pop_back_val();
- Visited.insert(Node->getEntry());
+ BasicBlock *Entry = Node->getEntry();
+ Visited.insert(Entry);
+
+ // There is nothing to do for island blocks because they are handled by
+ // handleIsland and we think of their terminators as immutable here.
+ // A subregion whose entry happens to be an island must be wired normally, or
+ // everything inside it past the entry would be dropped.
+ if (!Node->isSubRegion() && isIsland(*Entry)) {
+ Tail.close(Node);
+ return;
+ }
+
+ // A callbr forwarder, a simple single-branch block reached straight from the
+ // callbr (created by handleIsland's edge split, or a reused trivial target),
+ // has already been converged into the dispatch ladder. Close the tail and
+ // leave it, so the normal machinery does not redirect it out of the ladder.
+ bool HasIslandPred = llvm::any_of(
+ predecessors(Entry), [](BasicBlock *Pred) { return isIsland(*Pred); });
+ if (HasIslandPred && isIntermediateTarget(*Entry)) {
+ Tail.close(Node);
+ return;
+ }
if (isPredictableTrue(Node)) {
- // Just a linear flow
- if (PrevNode) {
- changeExit(PrevNode, Node->getEntry(), true);
- }
- PrevNode = Node;
+ // Just a linear flow. Only a rewritable tail is redirected to flow into
+ // this node; a closed tail is an island boundary handleIsland already
+ // wired, so it is left as-is.
+ if (Tail.isRewritable())
+ changeExit(Tail, Entry, true);
+ Tail = Node;
} else {
// Insert extra prefix node (or reuse last one)
auto [Flow, DL] = needPrefix(false);
// Insert extra postfix node (or use exit instead)
- BasicBlock *Entry = Node->getEntry();
BasicBlock *Next = needPostfix(Flow, ExitUseAllowed);
// let it point to entry and next block
@@ -1188,17 +1311,200 @@ void StructurizeCFG::wireFlow(bool ExitUseAllowed,
addPhiValues(Flow, Entry);
DT->changeImmediateDominator(Entry, Flow);
- PrevNode = Node;
+ Tail = Node;
while (!Order.empty() && !Visited.count(LoopEnd) &&
dominatesPredicates(Entry, Order.back())) {
handleLoops(false, LoopEnd);
}
- changeExit(PrevNode, Next, false);
+ if (Tail.isRewritable())
+ changeExit(Tail, Next, false);
setPrevNode(Next);
}
}
+/// Handle an "island", i.e. a block whose terminator is not rewritable (e.g., a
+/// callbr) in this context and considered immutable from this pass's
+/// perspective.
+///
+/// The island is left untouched; its edges are split into forwarders that
+/// converge at a new Flow block (ExitFlow). The callbr's runtime target choice
+/// is recovered as a per-target i1 "sel" phi and re-dispatched by a structured
+/// ladder of 2-way branches:
+///
+/// callbr
+/// / | \
+/// fwd_0 fwd_1 fwd_2 forwarders (intermediate target blocks)
+/// \ | /
+/// ExitFlow (Flow_0) -- sel_0? --> real_0
+/// | else
+/// Flow_1 --------- sel_1? --> real_1
+/// | else
+/// real_2
+/// or Flow_2 (if real_1 == real_2) ------> real_2 (shared last
+/// target)
+///
+void StructurizeCFG::handleIsland(BasicBlock *IslandBB) {
+ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
+
+ // 1. Split the island's edges into forwarders, so the island's targets are no
+ // longer direct successors of the (immutable) callbr terminator. This also
+ // turns a self-loop back-edge into a *mutable* forwarder, so later steps
+ // never rewrite the island edge itself.
+ //
+ // Reuse a target as-is only if it is already a trivial forwarder *inside this
+ // region*. A trivial target that is the region's exit block is still split:
+ // the ladder consumes its forwarders, and consuming the exit block would move
+ // the region's SESE boundary, which the precomputed RegionInfo cannot
+ // represent.
+ SmallVector<BasicBlock *, 4> Targets;
+ if (CallBrInst *CallBr = dyn_cast<CallBrInst>(IslandBB->getTerminator())) {
+ for (unsigned I = 0; I < CallBr->getNumSuccessors(); ++I) {
+ BasicBlock *Target = CallBr->getSuccessor(I);
+ // Leave an unreachable target's callbr edge direct and keep the dead lane
+ // out of the dispatch sel phi, which SILowerI1Copies cannot lower. It
+ // stays a direct predecessor of its successor and is not part of the
+ // ladder.
+ // However, that gives the region exit a predecessor createFlow's
+ // single-pred exit wiring does not account for. Flag that the exit's
+ // immediate dominator needs to be fixed.
+ if (isUnreachableTarget(*Target)) {
+ NeedFixRegionExitDom = true;
+ continue;
+ }
+ if (isIntermediateTarget(*Target) && ParentRegion->contains(Target)) {
+ Targets.push_back(Target);
+ continue;
+ }
+ BasicBlock *NewBB = SplitCallBrEdge(IslandBB, Target, I, &DTU);
+ ParentRegion->getRegionInfo()->setRegionFor(NewBB, ParentRegion);
+ Targets.push_back(NewBB);
+ }
+ } else {
+ llvm_unreachable("not a supported island type");
+ }
+
+ // 2. Create EntryFlow in front of the island for its in-region, non-forwarder
+ // predecessors (e.g. a loop back-edge from a separate latch), giving a
+ // rewritable header anchor. Forwarders route through the ladder instead.
+ // Out-of-region predecessors (only the region entry has any, by SESE) stay
+ // direct, keeping the region boundary intact. Skipped when empty to avoid an
+ // orphan Flow.
+ SmallVector<BasicBlock *, 4> Preds;
+ for (BasicBlock *Pred : predecessors(IslandBB))
+ if (ParentRegion->contains(Pred) && !llvm::is_contained(Targets, Pred))
+ Preds.push_back(Pred);
+ if (!Preds.empty()) {
+ BasicBlock *EntryFlow = SplitBlockPredecessors(IslandBB, Preds, "", &DTU);
+ EntryFlow->setName(FlowBlockName);
+ FlowSet.insert(EntryFlow);
+ ParentRegion->getRegionInfo()->setRegionFor(EntryFlow, ParentRegion);
+ }
+
+ // 3. Build the dispatch ladder shown in the diagram above. sel_k is the i1
+ // phi in ExitFlow that is true only on the edge from forwarder k; ExitFlow
+ // dominates the whole ladder, so the sel phis are usable at every level. If
+ // the island has a self-loop, it is dispatched by the ladder as well.
+ //
+ // Edge edits are recorded in DTUpdates and applied via the same
+ // DomTreeUpdater, keeping the DominatorTree incrementally correct. The island
+ // Flow blocks are created without DT->addNewBlock and get their idoms from
+ // the inserted edges, so we must not mix in a direct getNextFlow here.
+ unsigned N = Targets.size();
+ SmallVector<DominatorTree::UpdateType, 16> DTUpdates;
+ auto MakeFlow = [&]() {
+ BasicBlock *Flow = BasicBlock::Create(Func->getContext(), FlowBlockName,
+ Func, ParentRegion->getExit());
+ FlowSet.insert(Flow);
+ ParentRegion->getRegionInfo()->setRegionFor(Flow, ParentRegion);
+ return Flow;
+ };
+ BasicBlock *ExitFlow = MakeFlow();
+
+ // Converge every forwarder into ExitFlow and remember the real target each
+ // one guards (its single successor). Real's phi incomings from the forwarder
+ // are re-bound onto the ladder edge in the dispatch loop below: we maintain
+ // SSA directly because handleIsland runs before createFlow, which clears the
+ // deferred delPhiValues/addPhiValues bookkeeping.
+ SmallVector<BasicBlock *, 4> RealTargets;
+ RealTargets.reserve(N);
+ for (BasicBlock *Fwd : Targets) {
+ BasicBlock *Real = Fwd->getSingleSuccessor();
+ assert(Real && "forwarder must have a single successor");
+ RealTargets.push_back(Real);
+
+ DebugLoc DL = Fwd->getTerminator()->getDebugLoc();
+ Fwd->getTerminator()->eraseFromParent();
+ UncondBrInst::Create(ExitFlow, Fwd)->setDebugLoc(DL);
+ DTUpdates.push_back({DominatorTree::Delete, Fwd, Real});
+ DTUpdates.push_back({DominatorTree::Insert, Fwd, ExitFlow});
+ }
+
+ // Materialize the per-target selection booleans in ExitFlow: sel_k is true
+ // exactly on the edge coming from Targets[k]. The last target needs no
+ // sel: it is the unconditional "else" of the ladder.
+ SmallVector<PHINode *, 4> Sel;
+ Sel.reserve(N ? N - 1 : 0);
+ for (unsigned I = 0; I + 1 < N; ++I) {
+ PHINode *P = PHINode::Create(Boolean, N, "island.sel");
+ P->insertInto(ExitFlow, ExitFlow->end());
+ for (unsigned J = 0; J < N; ++J)
+ P->addIncoming(J == I ? BoolTrue : BoolFalse, Targets[J]);
+ Sel.push_back(P);
+ }
+
+ // Wire each ladder level's 2-way branch to its real target.
+ BasicBlock *CurFlow = ExitFlow;
+ auto RebindRealPhis = [](BasicBlock *Real, BasicBlock *Fwd,
+ BasicBlock *NewPred) {
+ for (PHINode &Phi : Real->phis()) {
+ int Idx = Phi.getBasicBlockIndex(Fwd);
+ if (Idx != -1)
+ Phi.setIncomingBlock(Idx, NewPred);
+ }
+ };
+ for (unsigned I = 0; I < N; ++I) {
+ BasicBlock *Fwd = Targets[I];
+ BasicBlock *Real = RealTargets[I];
+
+ // Real is now entered from CurFlow instead of from its forwarder. Re-bind
+ // Real's phi incomings from Fwd onto the new CurFlow edge, keeping the
+ // original value. This preserves phis that distinguish island targets (e.g.
+ // a join phi selecting the callbr's =r output on the fallthrough vs another
+ // value on an indirect target). The value was defined before the callbr (or
+ // is the callbr's own output), so it dominates CurFlow.
+ RebindRealPhis(Real, Fwd, CurFlow);
+
+ if (I + 1 < N) {
+ BasicBlock *NextReal = RealTargets[I + 1];
+ if (I + 2 == N && Real != NextReal) {
+ // Optimization for last target: branch directly to the final real
+ // target when that keeps the two island lanes as distinct CFG edges.
+ // Shared targets need the final Flow block so PHIs can retain per-lane
+ // values.
+ RebindRealPhis(NextReal, Targets[I + 1], CurFlow);
+ CondBrInst::Create(Sel[I], Real, NextReal, CurFlow);
+ DTUpdates.push_back({DominatorTree::Insert, CurFlow, Real});
+ DTUpdates.push_back({DominatorTree::Insert, CurFlow, NextReal});
+ break;
+ }
+
+ BasicBlock *NextFlow = MakeFlow();
+ CondBrInst::Create(Sel[I], Real, NextFlow, CurFlow);
+ DTUpdates.push_back({DominatorTree::Insert, CurFlow, Real});
+ DTUpdates.push_back({DominatorTree::Insert, CurFlow, NextFlow});
+ CurFlow = NextFlow;
+ } else {
+ // Last target: unconditional else of the ladder.
+ UncondBrInst::Create(Real, CurFlow);
+ DTUpdates.push_back({DominatorTree::Insert, CurFlow, Real});
+ }
+ }
+
+ // Apply all island edge edits to the DominatorTree at once.
+ DTU.applyUpdates(DTUpdates);
+}
+
void StructurizeCFG::handleLoops(bool ExitUseAllowed,
BasicBlock *LoopEnd) {
RegionNode *Node = Order.back();
@@ -1243,17 +1549,27 @@ void StructurizeCFG::createFlow() {
Conditions.clear();
LoopConds.clear();
- PrevNode = nullptr;
+ Tail = FlowTail();
Visited.clear();
while (!Order.empty()) {
handleLoops(EntryDominatesExit, nullptr);
}
- if (PrevNode)
- changeExit(PrevNode, Exit, EntryDominatesExit);
- else
+ if (Tail.isRewritable())
+ changeExit(Tail, Exit, EntryDominatesExit);
+ else if (Tail.isNone())
assert(EntryDominatesExit);
+
+ if (NeedFixRegionExitDom) {
+ BasicBlock *Exit = ParentRegion->getExit();
+ BasicBlock *Idom = nullptr;
+ for (BasicBlock *Pred : predecessors(Exit))
+ Idom = Idom ? DT->findNearestCommonDominator(Idom, Pred) : Pred;
+ if (Idom)
+ DT->changeImmediateDominator(Exit, Idom);
+ NeedFixRegionExitDom = false;
+ }
}
/// Handle a rare case where the disintegrated nodes instructions
@@ -1395,12 +1711,7 @@ bool StructurizeCFG::makeUniformRegion(Region *R, UniformityInfo &UA) {
/// Run the transformation for each region found
bool StructurizeCFG::run(Region *R, DominatorTree *DT,
const TargetTransformInfo *TTI) {
- // CallBr and its corresponding direct target blocks are for now ignored by
- // this pass. This is not a limitation for the currently intended uses cases
- // of callbr in the AMDGPU backend.
- // Parent and child regions are not affected by this (current) restriction.
- // See `llvm/test/Transforms/StructurizeCFG/callbr.ll` for details.
- if (R->isTopLevelRegion() || isa<CallBrInst>(R->getEntry()->getTerminator()))
+ if (R->isTopLevelRegion())
return false;
this->DT = DT;
@@ -1409,6 +1720,15 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT,
ParentRegion = R;
+ SmallVector<BasicBlock *, 8> IslandBlocks;
+ for (RegionNode *E : R->elements()) {
+ if (!E->isSubRegion() && isIsland(*E->getEntry()))
+ IslandBlocks.push_back(E->getNodeAs<BasicBlock>());
+ }
+
+ for (BasicBlock *BB : IslandBlocks)
+ handleIsland(BB);
+
orderNodes();
collectInfos();
createFlow();
diff --git a/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll
index 8ac31b3c70ed7..8da292a4930a1 100644
--- a/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll
@@ -35,7 +35,7 @@ define void @test_kill(ptr %src, ptr %dst, i1 %c) {
; GISEL-NEXT: s_cbranch_scc0 .LBB0_4
; GISEL-NEXT: ; %bb.1:
; GISEL-NEXT: s_and_b64 exec, exec, s[4:5]
-; GISEL-NEXT: ; %bb.2: ; %cont
+; GISEL-NEXT: ; %bb.2: ; %Flow
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GISEL-NEXT: flat_store_dword v[2:3], v0
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -91,7 +91,7 @@ define void @test_kill_block_order(ptr %src, ptr %dst, i1 %c) {
; GISEL-NEXT: s_cbranch_scc0 .LBB1_4
; GISEL-NEXT: ; %bb.1:
; GISEL-NEXT: s_and_b64 exec, exec, s[4:5]
-; GISEL-NEXT: ; %bb.2: ; %cont
+; GISEL-NEXT: ; %bb.2: ; %Flow
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GISEL-NEXT: flat_store_dword v[2:3], v0
; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll
index 253a6ec100eae..6e277de92aecc 100644
--- a/llvm/test/CodeGen/AMDGPU/callbr.ll
+++ b/llvm/test/CodeGen/AMDGPU/callbr.ll
@@ -7,20 +7,34 @@ define void @callbr_inline_asm(ptr %src, ptr %dst1, ptr %dst2, i32 %c) {
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: flat_load_dword v0, v[0:1]
; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: v_cmp_gt_i32 vcc v6, 42; s_cbranch_vccnz .LBB0_2
+; CHECK-NEXT: v_cmp_gt_i32 vcc v6, 42; s_cbranch_vccnz .LBB0_7
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; %bb.1: ; %fallthrough
+; CHECK-NEXT: ; %bb.1: ; %.target.fallthrough
+; CHECK-NEXT: s_mov_b64 s[4:5], -1
+; CHECK-NEXT: .LBB0_2: ; %Flow
+; CHECK-NEXT: s_xor_b64 s[4:5], s[4:5], -1
+; CHECK-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[6:7]
+; CHECK-NEXT: s_cbranch_execz .LBB0_4
+; CHECK-NEXT: ; %bb.3: ; %indirect
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: flat_store_dword v[4:5], v0
+; CHECK-NEXT: ; implicit-def: $vgpr2
+; CHECK-NEXT: .LBB0_4: ; %Flow1
+; CHECK-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
+; CHECK-NEXT: s_cbranch_execz .LBB0_6
+; CHECK-NEXT: ; %bb.5: ; %fallthrough
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: flat_store_dword v[2:3], v0
+; CHECK-NEXT: .LBB0_6: ; %ret
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
-; CHECK-NEXT: .LBB0_2: ; Inline asm indirect target
-; CHECK-NEXT: ; %indirect
+; CHECK-NEXT: .LBB0_7: ; Inline asm indirect target
+; CHECK-NEXT: ; %.target.indirect
; CHECK-NEXT: ; Label of block must be emitted
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_store_dword v[4:5], v0
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_setpc_b64 s[30:31]
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_branch .LBB0_2
%a = load i32, ptr %src, align 4
callbr void asm "v_cmp_gt_i32 vcc $0, 42; s_cbranch_vccnz ${1:l}", "r,!i"(i32 %c) to label %fallthrough [label %indirect]
fallthrough:
@@ -41,10 +55,14 @@ define void @callbr_self_loop(i1 %c) {
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: s_branch .LBB1_1
-; CHECK-NEXT: .LBB1_2: ; Inline asm indirect target
-; CHECK-NEXT: ; %callbr.target.ret
+; CHECK-NEXT: ; %bb.2: ; %callbr.target.callbr
+; CHECK-NEXT: ; in Loop: Header=BB1_1 Depth=1
+; CHECK-NEXT: s_cbranch_execnz .LBB1_1
+; CHECK-NEXT: s_branch .LBB1_4
+; CHECK-NEXT: .LBB1_3: ; Inline asm indirect target
+; CHECK-NEXT: ; %callbr.target.callbr.target.ret
; CHECK-NEXT: ; Label of block must be emitted
+; CHECK-NEXT: .LBB1_4: ; %ret
; CHECK-NEXT: s_setpc_b64 s[30:31]
br label %callbr
callbr:
diff --git a/llvm/test/CodeGen/AMDGPU/infinite-loop.ll b/llvm/test/CodeGen/AMDGPU/infinite-loop.ll
index df635925b87df..6b816b614456b 100644
--- a/llvm/test/CodeGen/AMDGPU/infinite-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/infinite-loop.ll
@@ -27,7 +27,6 @@ define amdgpu_kernel void @infinite_loop(ptr addrspace(1) %out) {
; IR-NEXT: br i1 true, label [[LOOP]], label [[DUMMYRETURNBLOCK:%.*]]
; IR: DummyReturnBlock:
; IR-NEXT: ret void
-;
entry:
br label %loop
@@ -45,9 +44,17 @@ define amdgpu_kernel void @infinite_loop_callbr(ptr addrspace(1) %out) {
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x3e7
+; SI-NEXT: s_and_b64 vcc, exec, -1
+; SI-NEXT: .LBB1_1: ; %loop
+; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ;;#ASMSTART
+; SI-NEXT: ;;#ASMEND
+; SI-NEXT: s_mov_b64 vcc, vcc
+; SI-NEXT: s_cbranch_vccnz .LBB1_1
+; SI-NEXT: ; %bb.2: ; %DummyReturnBlock
; SI-NEXT: s_endpgm
; IR-LABEL: @infinite_loop_callbr(
; IR-NEXT: entry:
@@ -61,7 +68,6 @@ define amdgpu_kernel void @infinite_loop_callbr(ptr addrspace(1) %out) {
; IR-NEXT: to label [[LOOP]] []
; IR: DummyReturnBlock:
; IR-NEXT: ret void
-;
entry:
callbr void asm "", ""() to label %loop []
@@ -101,7 +107,6 @@ define amdgpu_kernel void @infinite_loop_ret(ptr addrspace(1) %out) {
; IR-NEXT: br i1 true, label [[LOOP]], label [[UNIFIEDRETURNBLOCK]]
; IR: UnifiedReturnBlock:
; IR-NEXT: ret void
-;
entry:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
%cond = icmp eq i32 %tmp, 1
@@ -122,18 +127,34 @@ define amdgpu_kernel void @infinite_loop_ret_callbr(ptr addrspace(1) %out) {
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SI-NEXT: ;;#ASMSTART
; SI-NEXT: ;;#ASMEND
-; SI-NEXT: ; %bb.1: ; %loop.preheader
+; SI-NEXT: ; %bb.1: ; %entry.target.loop.preheader
+; SI-NEXT: s_mov_b64 s[0:1], -1
+; SI-NEXT: .LBB3_2: ; %Flow4
+; SI-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
+; SI-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; SI-NEXT: s_cbranch_execz .LBB3_5
+; SI-NEXT: ; %bb.3: ; %loop.preheader
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x3e7
+; SI-NEXT: s_and_b64 vcc, exec, -1
+; SI-NEXT: .LBB3_4: ; %loop
+; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: .LBB3_2: ; Inline asm indirect target
-; SI-NEXT: ; %UnifiedReturnBlock
-; SI-NEXT: ; Label of block must be emitted
+; SI-NEXT: ;;#ASMSTART
+; SI-NEXT: ;;#ASMEND
+; SI-NEXT: s_mov_b64 vcc, vcc
+; SI-NEXT: s_cbranch_vccnz .LBB3_4
+; SI-NEXT: .LBB3_5: ; %UnifiedReturnBlock
; SI-NEXT: s_endpgm
+; SI-NEXT: .LBB3_6: ; Inline asm indirect target
+; SI-NEXT: ; %entry.target.UnifiedReturnBlock
+; SI-NEXT: ; Label of block must be emitted
+; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_branch .LBB3_2
; IR-LABEL: @infinite_loop_ret_callbr(
; IR-NEXT: entry:
; IR-NEXT: [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -149,7 +170,6 @@ define amdgpu_kernel void @infinite_loop_ret_callbr(ptr addrspace(1) %out) {
; IR-NEXT: to label [[LOOP]] []
; IR: UnifiedReturnBlock:
; IR-NEXT: ret void
-;
entry:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
%cond = icmp eq i32 %tmp, 1
@@ -214,7 +234,6 @@ define amdgpu_kernel void @infinite_loops(ptr addrspace(1) %out) {
; IR-NEXT: br i1 true, label [[LOOP2]], label [[DUMMYRETURNBLOCK]]
; IR: DummyReturnBlock:
; IR-NEXT: ret void
-;
entry:
br i1 poison, label %loop1, label %loop2
@@ -234,22 +253,51 @@ define amdgpu_kernel void @infinite_loops_callbr(ptr addrspace(1) %out) {
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: ;;#ASMSTART
; SI-NEXT: ;;#ASMEND
-; SI-NEXT: ; %bb.1: ; %loop1
+; SI-NEXT: ; %bb.1: ; %loop1.preheader
+; SI-NEXT: s_mov_b64 s[2:3], -1
+; SI-NEXT: .LBB5_2: ; %Flow9
+; SI-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; SI-NEXT: s_mov_b64 s[2:3], -1
+; SI-NEXT: s_cbranch_vccz .LBB5_6
+; SI-NEXT: ; %bb.3:
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, 0x3e7
+; SI-NEXT: v_mov_b32_e32 v0, 0x378
+; SI-NEXT: s_and_b64 vcc, exec, -1
+; SI-NEXT: .LBB5_4: ; %loop2
+; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: s_endpgm
-; SI-NEXT: .LBB5_2: ; Inline asm indirect target
-; SI-NEXT: ; %loop2.preheader
-; SI-NEXT: ; Label of block must be emitted
+; SI-NEXT: ;;#ASMSTART
+; SI-NEXT: ;;#ASMEND
+; SI-NEXT: s_mov_b64 vcc, vcc
+; SI-NEXT: s_cbranch_vccnz .LBB5_4
+; SI-NEXT: ; %bb.5: ; %Flow4
+; SI-NEXT: s_mov_b64 s[2:3], 0
+; SI-NEXT: .LBB5_6: ; %Flow10
+; SI-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; SI-NEXT: s_cbranch_vccnz .LBB5_9
+; SI-NEXT: ; %bb.7:
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, 0x378
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, 0x3e7
+; SI-NEXT: s_and_b64 vcc, exec, -1
+; SI-NEXT: .LBB5_8: ; %loop1
+; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: ;;#ASMSTART
+; SI-NEXT: ;;#ASMEND
+; SI-NEXT: s_mov_b64 vcc, vcc
+; SI-NEXT: s_cbranch_vccnz .LBB5_8
+; SI-NEXT: .LBB5_9: ; %DummyReturnBlock
; SI-NEXT: s_endpgm
+; SI-NEXT: .LBB5_10: ; Inline asm indirect target
+; SI-NEXT: ; %loop2.preheader
+; SI-NEXT: ; Label of block must be emitted
+; SI-NEXT: s_mov_b64 s[2:3], 0
+; SI-NEXT: s_branch .LBB5_2
; IR-LABEL: @infinite_loops_callbr(
; IR-NEXT: entry:
; IR-NEXT: callbr void asm "", "r,!i"(i32 poison)
@@ -268,7 +316,6 @@ define amdgpu_kernel void @infinite_loops_callbr(ptr addrspace(1) %out) {
; IR-NEXT: to label [[LOOP2:%.*]] []
; IR: DummyReturnBlock:
; IR-NEXT: ret void
-;
entry:
callbr void asm "", "r,!i"(i32 poison) to label %loop1 [label %loop2]
@@ -329,7 +376,6 @@ define amdgpu_kernel void @infinite_loop_nest_ret(ptr addrspace(1) %out) {
; IR-NEXT: br i1 [[COND3]], label [[INNER_LOOP]], label [[OUTER_LOOP]]
; IR: UnifiedReturnBlock:
; IR-NEXT: ret void
-;
entry:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
%cond1 = icmp ne i32 %tmp, 1 ; avoid following BB optimizing away through the domination
@@ -353,38 +399,67 @@ define amdgpu_kernel void @infinite_loop_nest_ret_callbr(ptr addrspace(1) %out)
; SI-LABEL: infinite_loop_nest_ret_callbr:
; SI: ; %bb.0: ; %entry
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 1, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; SI-NEXT: ;;#ASMSTART
; SI-NEXT: ;;#ASMEND
-; SI-NEXT: ; %bb.1: ; %outer_loop.preheader
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v0, 0x3e7
-; SI-NEXT: s_and_b64 s[0:1], exec, 0
-; SI-NEXT: s_branch .LBB7_3
-; SI-NEXT: .LBB7_2: ; %loop.exit.guard
-; SI-NEXT: ; in Loop: Header=BB7_3 Depth=1
-; SI-NEXT: s_and_b64 vcc, exec, s[2:3]
-; SI-NEXT: s_cbranch_vccnz .LBB7_5
-; SI-NEXT: .LBB7_3: ; %outer_loop
-; SI-NEXT: ; =>This Inner Loop Header: Depth=1
+; SI-NEXT: ; %bb.1: ; %entry.target.outer_loop.preheader
+; SI-NEXT: s_mov_b64 s[0:1], -1
+; SI-NEXT: .LBB7_2: ; %Flow7
+; SI-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
+; SI-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; SI-NEXT: s_cbranch_execz .LBB7_11
+; SI-NEXT: ; %bb.3: ; %outer_loop.preheader
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v1, 0x3e7
+; SI-NEXT: .LBB7_4: ; %outer_loop
+; SI-NEXT: ; =>This Loop Header: Depth=1
+; SI-NEXT: ; Child Loop BB7_5 Depth 2
; SI-NEXT: ;;#ASMSTART
; SI-NEXT: ;;#ASMEND
+; SI-NEXT: s_mov_b64 s[4:5], 0
+; SI-NEXT: .LBB7_5: ; %inner_loop
+; SI-NEXT: ; Parent Loop BB7_4 Depth=1
+; SI-NEXT: ; => This Inner Loop Header: Depth=2
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: s_mov_b64 s[2:3], -1
-; SI-NEXT: s_mov_b64 vcc, s[0:1]
-; SI-NEXT: s_cbranch_vccz .LBB7_2
-; SI-NEXT: ; %bb.4: ; %TransitionBlock.target.outer_loop
-; SI-NEXT: ; in Loop: Header=BB7_3 Depth=1
-; SI-NEXT: s_mov_b64 s[2:3], 0
-; SI-NEXT: s_branch .LBB7_2
-; SI-NEXT: .LBB7_5: ; Inline asm indirect target
-; SI-NEXT: ; %UnifiedReturnBlock
+; SI-NEXT: ;;#ASMSTART
+; SI-NEXT: ;;#ASMEND
+; SI-NEXT: ; %bb.6: ; %TransitionBlock.target.inner_loop
+; SI-NEXT: ; in Loop: Header=BB7_5 Depth=2
+; SI-NEXT: s_mov_b64 s[6:7], -1
+; SI-NEXT: .LBB7_7: ; %Flow2
+; SI-NEXT: ; in Loop: Header=BB7_5 Depth=2
+; SI-NEXT: s_xor_b64 s[6:7], s[6:7], -1
+; SI-NEXT: s_and_b64 s[6:7], exec, s[6:7]
+; SI-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
+; SI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SI-NEXT: s_cbranch_execnz .LBB7_5
+; SI-NEXT: s_branch .LBB7_9
+; SI-NEXT: .LBB7_8: ; Inline asm indirect target
+; SI-NEXT: ; %TransitionBlock.target.outer_loop
+; SI-NEXT: ; in Loop: Header=BB7_5 Depth=2
; SI-NEXT: ; Label of block must be emitted
+; SI-NEXT: s_mov_b64 s[6:7], 0
+; SI-NEXT: s_branch .LBB7_7
+; SI-NEXT: .LBB7_9: ; %loop.exit.guard
+; SI-NEXT: ; in Loop: Header=BB7_4 Depth=1
+; SI-NEXT: s_or_b64 exec, exec, s[4:5]
+; SI-NEXT: ; %bb.10: ; %Flow4
+; SI-NEXT: ; in Loop: Header=BB7_4 Depth=1
+; SI-NEXT: s_mov_b64 vcc, 0
+; SI-NEXT: s_branch .LBB7_4
+; SI-NEXT: .LBB7_11: ; %UnifiedReturnBlock
; SI-NEXT: s_endpgm
+; SI-NEXT: .LBB7_12: ; Inline asm indirect target
+; SI-NEXT: ; %entry.target.UnifiedReturnBlock
+; SI-NEXT: ; Label of block must be emitted
+; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_branch .LBB7_2
; IR-LABEL: @infinite_loop_nest_ret_callbr(
; IR-NEXT: entry:
; IR-NEXT: [[TMP:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -405,7 +480,6 @@ define amdgpu_kernel void @infinite_loop_nest_ret_callbr(ptr addrspace(1) %out)
; IR-NEXT: to label [[INNER_LOOP]] [label %outer_loop]
; IR: UnifiedReturnBlock:
; IR-NEXT: ret void
-;
entry:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
%cond1 = icmp ne i32 %tmp, 1 ; avoid following BB optimizing away through the domination
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-nested-control-flows.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-nested-control-flows.ll
index 01bcdad3fc220..a504984eed05e 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-nested-control-flows.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-nested-control-flows.ll
@@ -93,42 +93,73 @@ define void @nested_inf_loop_callbr(i32 %0, i32 %1) {
; ISA-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; ISA-NEXT: ;;#ASMSTART
; ISA-NEXT: ;;#ASMEND
-; ISA-NEXT: ; implicit-def: $sgpr6_sgpr7
-; ISA-NEXT: ; implicit-def: $sgpr4_sgpr5
-; ISA-NEXT: .LBB1_1: ; %BB1
-; ISA-NEXT: ; =>This Inner Loop Header: Depth=1
+; ISA-NEXT: s_branch .LBB1_2
+; ISA-NEXT: .LBB1_1: ; %Flow14
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
+; ISA-NEXT: s_and_b64 vcc, exec, s[4:5]
+; ISA-NEXT: s_cbranch_vccnz .LBB1_14
+; ISA-NEXT: .LBB1_2: ; %BB1
+; ISA-NEXT: ; =>This Loop Header: Depth=1
+; ISA-NEXT: ; Child Loop BB1_6 Depth 2
; ISA-NEXT: ;;#ASMSTART
; ISA-NEXT: ;;#ASMEND
-; ISA-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
-; ISA-NEXT: s_and_b64 s[8:9], s[4:5], exec
-; ISA-NEXT: s_or_b64 s[6:7], s[6:7], s[8:9]
-; ISA-NEXT: .LBB1_2: ; %BB3
-; ISA-NEXT: ; in Loop: Header=BB1_1 Depth=1
+; ISA-NEXT: ; %bb.3: ; %BB1.target.BB3
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
+; ISA-NEXT: s_mov_b64 s[4:5], -1
+; ISA-NEXT: .LBB1_4: ; %Flow5
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
+; ISA-NEXT: s_xor_b64 s[4:5], s[4:5], -1
+; ISA-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
+; ISA-NEXT: s_xor_b64 s[4:5], exec, s[6:7]
+; ISA-NEXT: s_cbranch_execz .LBB1_11
+; ISA-NEXT: ; %bb.5: ; %BB2
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
; ISA-NEXT: ;;#ASMSTART
; ISA-NEXT: ;;#ASMEND
-; ISA-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
-; ISA-NEXT: s_and_b64 s[8:9], s[6:7], exec
-; ISA-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; ISA-NEXT: s_branch .LBB1_1
-; ISA-NEXT: .LBB1_3: ; Inline asm indirect target
-; ISA-NEXT: ; %BB2
-; ISA-NEXT: ; in Loop: Header=BB1_1 Depth=1
+; ISA-NEXT: s_mov_b64 s[6:7], 0
+; ISA-NEXT: .LBB1_6: ; %TransitionBlock
+; ISA-NEXT: ; Parent Loop BB1_2 Depth=1
+; ISA-NEXT: ; => This Inner Loop Header: Depth=2
+; ISA-NEXT: ;;#ASMSTART
+; ISA-NEXT: ;;#ASMEND
+; ISA-NEXT: ; %bb.7: ; %TransitionBlock.target.BB3
+; ISA-NEXT: ; in Loop: Header=BB1_6 Depth=2
+; ISA-NEXT: s_mov_b64 s[8:9], -1
+; ISA-NEXT: .LBB1_8: ; %Flow2
+; ISA-NEXT: ; in Loop: Header=BB1_6 Depth=2
+; ISA-NEXT: s_and_b64 s[8:9], exec, s[8:9]
+; ISA-NEXT: s_or_b64 s[6:7], s[8:9], s[6:7]
+; ISA-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; ISA-NEXT: s_cbranch_execnz .LBB1_6
+; ISA-NEXT: s_branch .LBB1_10
+; ISA-NEXT: .LBB1_9: ; Inline asm indirect target
+; ISA-NEXT: ; %TransitionBlock.target.BB4
+; ISA-NEXT: ; in Loop: Header=BB1_6 Depth=2
; ISA-NEXT: ; Label of block must be emitted
+; ISA-NEXT: s_mov_b64 s[8:9], 0
+; ISA-NEXT: s_branch .LBB1_8
+; ISA-NEXT: .LBB1_10: ; %loop.exit.guard
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
+; ISA-NEXT: s_or_b64 exec, exec, s[6:7]
+; ISA-NEXT: .LBB1_11: ; %Flow13
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
+; ISA-NEXT: s_or_b64 exec, exec, s[4:5]
+; ISA-NEXT: s_mov_b64 vcc, exec
+; ISA-NEXT: s_mov_b64 s[4:5], -1
+; ISA-NEXT: s_cbranch_execz .LBB1_1
+; ISA-NEXT: ; %bb.12: ; %BB3
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
; ISA-NEXT: ;;#ASMSTART
; ISA-NEXT: ;;#ASMEND
-; ISA-NEXT: s_mov_b64 s[6:7], -1
-; ISA-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; ISA-NEXT: s_cbranch_execz .LBB1_5
-; ISA-NEXT: ; %bb.4: ; %TransitionBlock.target.BB3
-; ISA-NEXT: ; in Loop: Header=BB1_1 Depth=1
-; ISA-NEXT: s_xor_b64 s[6:7], exec, -1
-; ISA-NEXT: .LBB1_5: ; %loop.exit.guard
-; ISA-NEXT: ; in Loop: Header=BB1_1 Depth=1
-; ISA-NEXT: s_or_b64 exec, exec, s[8:9]
-; ISA-NEXT: s_and_b64 vcc, exec, s[6:7]
-; ISA-NEXT: s_mov_b64 s[6:7], 0
-; ISA-NEXT: s_cbranch_vccz .LBB1_2
-; ISA-NEXT: ; %bb.6: ; %DummyReturnBlock
+; ISA-NEXT: s_mov_b64 s[4:5], 0
+; ISA-NEXT: s_branch .LBB1_1
+; ISA-NEXT: .LBB1_13: ; Inline asm indirect target
+; ISA-NEXT: ; %BB1.target.BB2
+; ISA-NEXT: ; in Loop: Header=BB1_2 Depth=1
+; ISA-NEXT: ; Label of block must be emitted
+; ISA-NEXT: s_mov_b64 s[4:5], 0
+; ISA-NEXT: s_branch .LBB1_4
+; ISA-NEXT: .LBB1_14: ; %DummyReturnBlock
; ISA-NEXT: s_setpc_b64 s[30:31]
BB:
callbr void asm "", ""() to label %BB1 []
diff --git a/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll b/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
index 004c27971131d..16722a4c55a1d 100644
--- a/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
@@ -96,7 +96,6 @@ define amdgpu_kernel void @kernel(i32 %a, ptr addrspace(1) %x, i32 noundef %n) {
; UNIFY-NEXT: br label [[IF_END6]]
; UNIFY: if.end6:
; UNIFY-NEXT: ret void
-;
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%cmp = icmp eq i32 %n, 256
@@ -134,55 +133,118 @@ if.end6:
define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 noundef %n) {
; CHECK-LABEL: kernel_callbr:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: s_load_dword s1, s[8:9], 0x10
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
+; CHECK-NEXT: s_load_dword s0, s[8:9], 0x10
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_cmpk_eq_i32 s1, 0x100
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_cmpk_eq_i32 s0, 0x100
+; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ; %bb.1: ; %entry.target.if.then
+; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: .LBB1_2: ; %Flow
+; CHECK-NEXT: s_load_dword s10, s[8:9], 0x0
+; CHECK-NEXT: s_andn2_b64 vcc, exec, s[0:1]
+; CHECK-NEXT: s_cbranch_vccz .LBB1_11
+; CHECK-NEXT: ; %bb.3: ; %if.else
+; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 10, v0
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; %bb.1: ; %if.then
-; CHECK-NEXT: s_cmp_eq_u32 s0, 0
+; CHECK-NEXT: ; %bb.4: ; %if.else.target.if.then3
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
+; CHECK-NEXT: .LBB1_5: ; %Flow11
+; CHECK-NEXT: s_mov_b64 s[2:3], 0
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_and_saveexec_b64 s[12:13], s[6:7]
+; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[12:13]
+; CHECK-NEXT: s_cbranch_execz .LBB1_10
+; CHECK-NEXT: ; %bb.6: ; %if.then3
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_cmp_eq_u32 s10, 0
+; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ; %bb.7: ; %if.then3.target.Flow7
+; CHECK-NEXT: .LBB1_8:
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_mov_b64 s[4:5], -1
+; CHECK-NEXT: .LBB1_9: ; %Flow19
+; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: .LBB1_10: ; %Flow18
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[2:3]
+; CHECK-NEXT: s_cbranch_vccnz .LBB1_12
+; CHECK-NEXT: s_branch .LBB1_15
+; CHECK-NEXT: .LBB1_11:
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_cbranch_execz .LBB1_15
+; CHECK-NEXT: .LBB1_12: ; %if.then
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_cmp_eq_u32 s10, 0
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: .LBB1_2: ; %if.end6.sink.split
-; CHECK-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x8
+; CHECK-NEXT: ; %bb.13: ; %if.then.target.if.end6.sink.split
+; CHECK-NEXT: s_mov_b64 s[2:3], -1
+; CHECK-NEXT: .LBB1_14: ; %Flow4
+; CHECK-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; CHECK-NEXT: s_mov_b64 s[4:5], -1
+; CHECK-NEXT: s_cbranch_vccnz .LBB1_20
+; CHECK-NEXT: .LBB1_15: ; %Flow20
+; CHECK-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
+; CHECK-NEXT: .LBB1_16: ; %UnifiedUnreachableBlock
+; CHECK-NEXT: ; divergent unreachable
+; CHECK-NEXT: .LBB1_17: ; %Flow22
+; CHECK-NEXT: s_or_b64 exec, exec, s[2:3]
+; CHECK-NEXT: s_and_saveexec_b64 s[0:1], s[4:5]
+; CHECK-NEXT: s_cbranch_execz .LBB1_19
+; CHECK-NEXT: ; %bb.18: ; %if.end6.sink.split
+; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; CHECK-NEXT: v_mov_b32_e32 v1, s0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dword v0, v1, s[2:3]
+; CHECK-NEXT: v_mov_b32_e32 v1, s10
+; CHECK-NEXT: global_store_dword v0, v1, s[0:1]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: .LBB1_3: ; Inline asm indirect target
-; CHECK-NEXT: ; %UnifiedReturnBlock
-; CHECK-NEXT: ; Label of block must be emitted
+; CHECK-NEXT: .LBB1_19: ; %UnifiedReturnBlock
; CHECK-NEXT: s_endpgm
-; CHECK-NEXT: .LBB1_4: ; Inline asm indirect target
-; CHECK-NEXT: ; %if.else
+; CHECK-NEXT: .LBB1_20: ; %cond.false
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_or_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_trap 2
+; CHECK-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
+; CHECK-NEXT: s_cbranch_execnz .LBB1_16
+; CHECK-NEXT: s_branch .LBB1_17
+; CHECK-NEXT: .LBB1_21: ; Inline asm indirect target
+; CHECK-NEXT: ; %entry.target.if.else
; CHECK-NEXT: ; Label of block must be emitted
-; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 10, v0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: ; %bb.5: ; %if.then3
-; CHECK-NEXT: s_cmp_eq_u32 s0, 0
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; CHECK-NEXT: ;;#ASMSTART
-; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
; CHECK-NEXT: s_branch .LBB1_2
-; CHECK-NEXT: .LBB1_6: ; Inline asm indirect target
-; CHECK-NEXT: ; %cond.false.i8
+; CHECK-NEXT: .LBB1_22: ; Inline asm indirect target
+; CHECK-NEXT: ; %if.else.target.UnifiedReturnBlock
; CHECK-NEXT: ; Label of block must be emitted
-; CHECK-NEXT: .LBB1_7: ; Inline asm indirect target
-; CHECK-NEXT: ; %cond.false
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
+; CHECK-NEXT: s_branch .LBB1_5
+; CHECK-NEXT: .LBB1_23: ; Inline asm indirect target
+; CHECK-NEXT: ; %if.then.target.cond.false
; CHECK-NEXT: ; Label of block must be emitted
+; CHECK-NEXT: s_mov_b64 s[2:3], 0
+; CHECK-NEXT: s_branch .LBB1_14
+; CHECK-NEXT: .LBB1_24: ; Inline asm indirect target
+; CHECK-NEXT: ; %if.then3.target.cond.false.i8
+; CHECK-NEXT: ; Label of block must be emitted
+; CHECK-NEXT: s_cbranch_execz .LBB1_8
+; CHECK-NEXT: ; %bb.25: ; %cond.false.i8
+; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_trap 2
-; CHECK-NEXT: ; divergent unreachable
-; CHECK-NEXT: s_branch .LBB1_3
+; CHECK-NEXT: s_branch .LBB1_9
; UNIFY-LABEL: @kernel_callbr(
; UNIFY-NEXT: entry:
; UNIFY-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -218,7 +280,6 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; UNIFY-NEXT: to label [[IF_END6:%.*]] []
; UNIFY: if.end6:
; UNIFY-NEXT: ret void
-;
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%cmp = icmp eq i32 %n, 256
diff --git a/llvm/test/Transforms/StructurizeCFG/callbr.ll b/llvm/test/Transforms/StructurizeCFG/callbr.ll
index 42f95194980d4..acf2f6e254148 100644
--- a/llvm/test/Transforms/StructurizeCFG/callbr.ll
+++ b/llvm/test/Transforms/StructurizeCFG/callbr.ll
@@ -1,102 +1,36 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=structurizecfg %s -o - | FileCheck %s
-
-; Structurize as usual, but don't tear callbr and its destination blocks apart.
-;
-; Note: currently, callbr blocks and their corresponding target blocks
-; themselves are not handled by the structurizer.* If the CFG turns out to be
-; unstructured at the end, the CFG lowering (si-annotate-control-flow) will
-; detect this. For the currently intended use cases of callbr in the context of
-; the AMDGPU backend, this is not a limitation (cf.
-; https://discourse.llvm.org/t/rfc-add-callbr-intrinsic-support/86087).
-;
-; Note 2: while callbr and its targets remain untouched, everything else is
-; handled as usual, even if it is nested in a callbr region.
-;
-; *FIXME: this will be fixed in the future. Callbr can be handled as follows:
-; Input IR:
-; ```
-; define void @foo_callbr() {
-; callbr void asm "", "!i"() to label %fallthrough [label %indirect, ...]
-; fallthrough:
-; br label %exit
-; indirect:
-; br label %exit
-; ...
-; exit:
-; ret void
-; }
-; ```
-;
-; Output IR:
-; ```
-; define void @foo_callbr() {
-; callbr void asm "", "!i"()
-; to label %fallthrough [label %fake.indirect, label %fake.indirect1, label %fake.indirect2, ...]
-; fake.indirect: ; preds = %0
-; br label %Flow
-; fake.indirect1: ; preds = %0
-; br label %Flow
-; fake.indirect2: ; preds = %0
-; br label %Flow
-; ...
-; Flow: ; preds = %fallthrough, %fake.indirect[0-N]
-; %1 = phi i1 [ false, %fallthrough ], [ true, %fake.indirect ], [ false, %fake.indirect[1-N] ]
-; br i1 %1, label %indirect, label %Flow1
-; Flow1: ; preds = %Flow, %indirect
-; %2 = phi i1 [ false, %Flow], [ true, %fake.indirect1 ], [ false, %indirect ]
-; br i1 %2, label %indirect1, label %Flow2
-; Flow2: ; preds = %Flow, %indirect1
-; %2 = phi i1 [ false, %Flow], [ true, %fake.indirect2 ], [ false, %indirect1 ]
-; br i1 %2, label %indirect2, label %Flow3
-; ...
-; fallthrough: ; preds = %0
-; br label %Flow
-; indirect: ; preds = %Flow
-; br label %Flow1
-; indirect1: ; preds = %Flow1
-; br label %Flow2
-; indirect2: : preds = %Flow2
-; br label %Flow3
-; ...
-; exit: ; preds = %indirectN, %FlowN
-; ret void
-; }
-; ```
-;
-; Output IR as ASCII-art:
-; %0
-; ---------------------
-; | | | |
-; v v v v
-; f f.i f.i1 f.i2
-; | | | |
-; v v v v
-; ---------------------
-; %Flow
-; | \
-; | %indirect
-; | /
-; %Flow1
-; | \
-; | %indirect1
-; | /
-; %Flow2
-; | \
-; | %indirect2
-; | /
-; %exit
+; RUN: opt -S -passes='structurizecfg,verify,verify<domtree>' %s -o - | FileCheck %s
+
+; StructurizeCFG handles callbr by leaving the callbr terminator itself
+; untouched and structurizing the surrounding CFG around it. Every callbr edge
+; (if it's reachable and not already in a trivial bypass shape, aka an
+; unconditional branch as the only instruction) is interposed with a bypass
+; block (via SplitCallBrEdge) so the targets are not direct successors of the
+; immutable callbr; the bypasses then converge into a Flow block whose
+; "callbr.sel" phis recover, as data, which target the callbr selected, and a
+; ladder of 2-way Flow branches re-dispatches to the targets. Loop back-edges
+; among the targets are routed so the normal loop machinery closes the loop. The
+; callbr terminator itself is never rewritten.
;
+; Tests are grouped by the structural situation they exercise.
+
+; ===== Single callbr, no surrounding control flow =====
; Only callbr, nothing to do.
define void @callbr_simple() {
; CHECK-LABEL: define void @callbr_simple() {
; CHECK-NEXT: [[CALLBR:.*:]]
; CHECK-NEXT: callbr void asm "", "!i"()
-; CHECK-NEXT: to label %[[INDIRECT:.*]] [label %indirect]
+; CHECK-NEXT: to label %[[FALLTHROUGH:.*]] [label %[[INDIRECT:.*]]]
+; CHECK: [[FALLTHROUGH]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
; CHECK: [[INDIRECT]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[INDIRECT1:.*:]]
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
+; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -111,24 +45,276 @@ exit:
ret void
}
+; callbr with no indirect targets at all.
+define void @callbr_no_indirect() {
+; CHECK-LABEL: define void @callbr_no_indirect() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", ""()
+; CHECK-NEXT: to label %[[FT:.*]] []
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void asm "", ""() to label %ft []
+ft:
+ br label %exit
+exit:
+ ret void
+}
+
+; callbr with many (4) indirect targets.
+define void @callbr_many_indirect(i1 %c) {
+; CHECK-LABEL: define void @callbr_many_indirect(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", "!i,!i,!i,!i"()
+; CHECK-NEXT: to label %[[FT:.*]] [label %[[I0:.*]], label %[[I1:.*]], label %[[I2:.*]], label %[[I3:.*]]]
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[I0]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[I1]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[I2]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[I3]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[I0]] ], [ false, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[CALLBR_SEL1:%.*]] = phi i1 [ false, %[[FT]] ], [ true, %[[I0]] ], [ false, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[CALLBR_SEL2:%.*]] = phi i1 [ false, %[[FT]] ], [ false, %[[I0]] ], [ true, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[CALLBR_SEL3:%.*]] = phi i1 [ false, %[[FT]] ], [ false, %[[I0]] ], [ false, %[[I1]] ], [ true, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[CALLBR_SEL3_INV:%.*]] = xor i1 [[CALLBR_SEL3]], true
+; CHECK-NEXT: [[CALLBR_SEL2_INV:%.*]] = xor i1 [[CALLBR_SEL2]], true
+; CHECK-NEXT: [[CALLBR_SEL1_INV:%.*]] = xor i1 [[CALLBR_SEL1]], true
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW4:.*]], label %[[FLOW8:.*]]
+; CHECK: [[FLOW4]]:
+; CHECK-NEXT: br i1 [[CALLBR_SEL1_INV]], label %[[FLOW5:.*]], label %[[FLOW9:.*]]
+; CHECK: [[FLOW8]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[FLOW5]]:
+; CHECK-NEXT: br i1 [[CALLBR_SEL2_INV]], label %[[FLOW6:.*]], label %[[FLOW10:.*]]
+; CHECK: [[FLOW9]]:
+; CHECK-NEXT: br label %[[FLOW8]]
+; CHECK: [[FLOW6]]:
+; CHECK-NEXT: br i1 [[CALLBR_SEL3_INV]], label %[[FLOW7:.*]], label %[[FLOW11:.*]]
+; CHECK: [[FLOW10]]:
+; CHECK-NEXT: br label %[[FLOW9]]
+; CHECK: [[FLOW7]]:
+; CHECK-NEXT: br label %[[FLOW11]]
+; CHECK: [[FLOW11]]:
+; CHECK-NEXT: br label %[[FLOW10]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void asm "", "!i,!i,!i,!i"() to label %ft [label %i0, label %i1, label %i2, label %i3]
+ft:
+ br label %exit
+i0:
+ br label %exit
+i1:
+ br label %exit
+i2:
+ br label %exit
+i3:
+ br label %exit
+exit:
+ ret void
+}
+
+; two callbr edges that share the same target block.
+define void @callbr_shared_target() {
+; CHECK-LABEL: define void @callbr_shared_target() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_FT:.*]] [label %[[ENTRY_TARGET_FT1:.*]]]
+; CHECK: [[FLOW:.*]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FT]] ], [ false, %[[ENTRY_TARGET_FT1]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW2:.*]], label %[[FT:.*]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: br label %[[FT]]
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_FT]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[ENTRY_TARGET_FT1]]:
+; CHECK-NEXT: br label %[[FLOW]]
+;
+entry:
+ callbr void asm "", "!i"() to label %ft [label %ft]
+ft:
+ br label %exit
+exit:
+ ret void
+}
+
+; One indirect target is already trivial (single unconditional branch);
+; handleIsland leaves it as a direct callbr destination and only splits the
+; non-trivial targets.
+define void @callbr_multi_indirect_one_trivial(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_multi_indirect_one_trivial(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", "!i,!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_FALLTHROUGH:.*]] [label %[[ENTRY_TARGET_INDIRECT0:.*]], label %[[INDIRECT1:.*]]]
+; CHECK: [[EXIT:.*]]:
+; CHECK-NEXT: br label %[[FLOW6:.*]]
+; CHECK: [[INDIRECT0:.*]]:
+; CHECK-NEXT: br i1 [[C]], label %[[A:.*]], label %[[FLOW1:.*]]
+; CHECK: [[A]]:
+; CHECK-NEXT: br label %[[FLOW1]]
+; CHECK: [[FLOW6]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[EXIT]] ], [ true, %[[FLOW2:.*]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[INDIRECT0]], label %[[FLOW8:.*]]
+; CHECK: [[FALLTHROUGH:.*]]:
+; CHECK-NEXT: br i1 [[D]], label %[[B:.*]], label %[[FLOW:.*]]
+; CHECK: [[B]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[INDIRECT1]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[FLOW8]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FALLTHROUGH]] ], [ false, %[[ENTRY_TARGET_INDIRECT0]] ], [ false, %[[INDIRECT1]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[ENTRY_TARGET_FALLTHROUGH]] ], [ true, %[[ENTRY_TARGET_INDIRECT0]] ], [ false, %[[INDIRECT1]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW4:.*]], label %[[FLOW6]]
+; CHECK: [[FLOW4]]:
+; CHECK-NEXT: br i1 [[TMP1]], label %[[FALLTHROUGH]], label %[[EXIT]]
+; CHECK: [[FLOW8]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_FALLTHROUGH]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[ENTRY_TARGET_INDIRECT0]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ callbr void asm "", "!i,!i"() to label %fallthrough [label %indirect0, label %indirect1]
+fallthrough:
+ br i1 %c, label %a, label %exit
+a:
+ br label %exit
+indirect0:
+ br i1 %d, label %b, label %exit
+b:
+ br label %exit
+indirect1:
+ br label %exit
+exit:
+ ret void
+}
+
+; ===== Intrinsic callees =====
+
+; Intrinsic callee (llvm.amdgcn.kill) with an unreachable indirect target.
+define void @callbr_kill_intrinsic(i1 %c) {
+; CHECK-LABEL: define void @callbr_kill_intrinsic(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[C]])
+; CHECK-NEXT: to label %[[CONT:.*]] [label %[[DEAD:.*]]]
+; CHECK: [[DEAD]]:
+; CHECK-NEXT: unreachable
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %dead]
+dead:
+ unreachable
+cont:
+ ret void
+}
+
+; Intrinsic callee whose dead target calls llvm.amdgcn.unreachable.
+define void @callbr_kill_with_amdgcn_unreachable(i1 %c) {
+; CHECK-LABEL: define void @callbr_kill_with_amdgcn_unreachable(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[C]])
+; CHECK-NEXT: to label %[[CONT:.*]] [label %[[DEAD:.*]]]
+; CHECK: [[DEAD]]:
+; CHECK-NEXT: call void @llvm.amdgcn.unreachable()
+; CHECK-NEXT: unreachable
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %dead]
+dead:
+ call void @llvm.amdgcn.unreachable()
+ unreachable
+cont:
+ ret void
+}
+
+; An amdgcn.unreachable dead-lane target that branches on to the exit (the shape
+; produced after exit unification). The callbr edge to it stays direct and the
+; dead lane is kept out of the dispatch, but it still reconverges at the exit.
+define void @callbr_kill_unreachable_with_successor(i1 %c) {
+; CHECK-LABEL: define void @callbr_kill_unreachable_with_successor(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[C]])
+; CHECK-NEXT: to label %[[CONT:.*]] [label %[[DEAD:.*]]]
+; CHECK: [[DEAD]]:
+; CHECK-NEXT: call void @llvm.amdgcn.unreachable()
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %dead]
+dead:
+ call void @llvm.amdgcn.unreachable()
+ br label %exit
+cont:
+ br label %exit
+exit:
+ ret void
+}
+
+; ===== callbr nested in / surrounded by non-callbr control flow =====
+
; Callbr nested in non-callbr: non-callbr is transformed
define void @callbr_in_non_callbr(i1 %c) {
; CHECK-LABEL: define void @callbr_in_non_callbr(
; CHECK-SAME: i1 [[C:%.*]]) {
; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
-; CHECK-NEXT: br i1 [[C_INV]], label %[[NOCALLBR:.*]], label %[[FLOW:.*]]
-; CHECK: [[FLOW]]:
+; CHECK-NEXT: br i1 [[C_INV]], label %[[NOCALLBR:.*]], label %[[FLOW2:.*]]
+; CHECK: [[FLOW2]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[NOCALLBR]] ], [ true, [[TMP0:%.*]] ]
; CHECK-NEXT: br i1 [[TMP1]], label %[[CALLBR:.*]], label %[[EXIT:.*]]
; CHECK: [[CALLBR]]:
; CHECK-NEXT: callbr void asm "", "!i"()
-; CHECK-NEXT: to label %[[INDIRECT:.*]] [label %indirect]
+; CHECK-NEXT: to label %[[FALLTHROUGH:.*]] [label %[[INDIRECT:.*]]]
+; CHECK: [[FALLTHROUGH]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
; CHECK: [[INDIRECT]]:
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[INDIRECT1:.*:]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[NOCALLBR]]:
; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[NOCALLBR]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[FLOW3:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -151,22 +337,27 @@ define void @non_callbr_in_callbr(i1 %c) {
; CHECK-SAME: i1 [[C:%.*]]) {
; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
; CHECK-NEXT: callbr void asm "", "!i"()
-; CHECK-NEXT: to label %[[INDIRECT:.*]] [label %indirect]
-; CHECK: [[INDIRECT]]:
+; CHECK-NEXT: to label %[[DOTTARGET_FALLTHROUGH:.*]] [label %[[INDIRECT:.*]]]
+; CHECK: [[FALLTHROUGH:.*]]:
; CHECK-NEXT: br i1 [[C_INV]], label %[[FALLTHROUGH2:.*]], label %[[FLOW:.*]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[FALLTHROUGH2]] ], [ true, %[[INDIRECT]] ]
-; CHECK-NEXT: br i1 [[TMP1]], label %[[FALLTHROUGH1:.*]], label %[[FLOW1:.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[FALLTHROUGH2]] ], [ true, %[[FALLTHROUGH]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[FALLTHROUGH1:.*]], label %[[FLOW1:.*]]
; CHECK: [[FALLTHROUGH1]]:
; CHECK-NEXT: br label %[[FLOW1]]
; CHECK: [[FALLTHROUGH2]]:
; CHECK-NEXT: br label %[[FLOW]]
-; CHECK: [[INDIRECT1:.*:]]
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[INDIRECT]]:
+; CHECK-NEXT: br label %[[FLOW2:.*]]
; CHECK: [[FLOW1]]:
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[DOTTARGET_FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[FALLTHROUGH]], label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
+; CHECK: [[_TARGET_FALLTHROUGH:.*:]]
+; CHECK-NEXT: br label %[[FLOW2]]
;
callbr void asm "", "!i"() to label %fallthrough [label %indirect]
fallthrough:
@@ -187,19 +378,22 @@ define void @callbr_nested_in_non_callbr(i1 %c, i1 %d, i1 %e, i1 %f) {
; CHECK-LABEL: define void @callbr_nested_in_non_callbr(
; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]], i1 [[E:%.*]], i1 [[F:%.*]]) {
; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
-; CHECK-NEXT: br i1 [[C_INV]], label %[[NOCALLBR:.*]], label %[[FLOW3:.*]]
-; CHECK: [[FLOW3]]:
+; CHECK-NEXT: br i1 [[C_INV]], label %[[NOCALLBR:.*]], label %[[FLOW7:.*]]
+; CHECK: [[FLOW7]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[FLOW:.*]] ], [ true, [[TMP0:%.*]] ]
; CHECK-NEXT: br i1 [[TMP1]], label %[[CALLBR:.*]], label %[[RET:.*]]
; CHECK: [[CALLBR]]:
; CHECK-NEXT: callbr void asm "", "!i"()
-; CHECK-NEXT: to label %[[INDIRECT:.*]] [label %indirect]
-; CHECK: [[INDIRECT]]:
+; CHECK-NEXT: to label %[[CALLBR_TARGET_FALLTHROUGH:.*]] [label %[[CALLBR_TARGET_INDIRECT:.*]]]
+; CHECK: [[FLOW5:.*]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[FLOW1:.*]] ], [ true, %[[FLOW3:.*]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[FALLTHROUGH:.*]], label %[[FLOW6:.*]]
+; CHECK: [[FALLTHROUGH]]:
; CHECK-NEXT: br i1 [[D]], label %[[FALLTHROUGH1:.*]], label %[[FLOW2:.*]]
; CHECK: [[FALLTHROUGH1]]:
; CHECK-NEXT: br label %[[FLOW2]]
-; CHECK: [[INDIRECT2:.*:]]
-; CHECK-NEXT: br i1 [[E]], label %[[INDIRECT1:.*]], label %[[FLOW1:.*]]
+; CHECK: [[FLOW4:.*]]:
+; CHECK-NEXT: br i1 [[E]], label %[[INDIRECT1:.*]], label %[[FLOW1]]
; CHECK: [[INDIRECT1]]:
; CHECK-NEXT: br label %[[FLOW1]]
; CHECK: [[NOCALLBR]]:
@@ -207,13 +401,23 @@ define void @callbr_nested_in_non_callbr(i1 %c, i1 %d, i1 %e, i1 %f) {
; CHECK: [[NOCALLBR1]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK-NEXT: br label %[[FLOW7]]
; CHECK: [[FLOW1]]:
-; CHECK-NEXT: br label %[[RET]]
+; CHECK-NEXT: br label %[[FLOW5]]
; CHECK: [[FLOW2]]:
+; CHECK-NEXT: br label %[[FLOW6]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CALLBR_TARGET_FALLTHROUGH]] ], [ false, %[[CALLBR_TARGET_INDIRECT]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW4]], label %[[FLOW5]]
+; CHECK: [[FLOW6]]:
; CHECK-NEXT: br label %[[RET]]
; CHECK: [[RET]]:
; CHECK-NEXT: ret void
+; CHECK: [[CALLBR_TARGET_FALLTHROUGH]]:
+; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK: [[CALLBR_TARGET_INDIRECT]]:
+; CHECK-NEXT: br label %[[FLOW3]]
;
br i1 %c, label %callbr, label %nocallbr
callbr:
@@ -233,3 +437,971 @@ nocallbr1:
ret:
ret void
}
+
+; non-trivial regions both before and after the island.
+define void @callbr_regions_around(i1 %c, i1 %d, i1 %e) {
+; CHECK-LABEL: define void @callbr_regions_around(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]], i1 [[E:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: [[E_INV:%.*]] = xor i1 [[E]], true
+; CHECK-NEXT: br i1 [[C_INV]], label %[[PRE2:.*]], label %[[FLOW3:.*]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[PRE2]] ], [ true, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRE1:.*]], label %[[PRECONT:.*]]
+; CHECK: [[PRE1]]:
+; CHECK-NEXT: br label %[[PRECONT]]
+; CHECK: [[PRE2]]:
+; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK: [[PRECONT]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[FT:.*]] [label %[[IND:.*]]]
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[FLOW1:.*]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW1]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW2:.*]], label %[[POST:.*]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: br label %[[POST]]
+; CHECK: [[POST]]:
+; CHECK-NEXT: br i1 [[E_INV]], label %[[POST2:.*]], label %[[FLOW:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[POST2]] ], [ true, %[[POST]] ]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[POST1:.*]], label %[[POSTCONT:.*]]
+; CHECK: [[POST1]]:
+; CHECK-NEXT: br label %[[POSTCONT]]
+; CHECK: [[POST2]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[POSTCONT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %pre1, label %pre2
+pre1:
+ br label %precont
+pre2:
+ br label %precont
+precont:
+ callbr void asm "", "!i"() to label %ft [label %ind]
+ft:
+ br label %post
+ind:
+ br label %post
+post:
+ br i1 %e, label %post1, label %post2
+post1:
+ br label %postcont
+post2:
+ br label %postcont
+postcont:
+ ret void
+}
+
+; callbr reached from the reconvergence of a non-trivial if-diamond subregion.
+define void @callbr_pred_from_subregion(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_pred_from_subregion(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: br i1 [[C_INV]], label %[[B:.*]], label %[[FLOW3:.*]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[B]] ], [ true, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[A:.*]], label %[[MERGE:.*]]
+; CHECK: [[A]]:
+; CHECK-NEXT: br i1 [[D]], label %[[A1:.*]], label %[[FLOW2:.*]]
+; CHECK: [[A1]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[B]]:
+; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CONT:.*]] [label %[[IND:.*]]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CONT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %A, label %B
+A:
+ br i1 %d, label %A1, label %merge
+A1:
+ br label %merge
+B:
+ br label %merge
+merge:
+ callbr void asm "", "!i"() to label %cont [label %ind]
+cont:
+ br label %exit
+ind:
+ br label %exit
+exit:
+ ret void
+}
+
+; ===== Structurizing the callbr targets' own subgraphs =====
+
+; a callbr fallthrough target contains its own if-diamond.
+define void @callbr_target_has_nested_region(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_target_has_nested_region(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_FT:.*]] [label %[[IND:.*]]]
+; CHECK: [[FT:.*]]:
+; CHECK-NEXT: br i1 [[C_INV]], label %[[T2:.*]], label %[[FLOW1:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[T2]] ], [ true, %[[FT]] ]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[T1:.*]], label %[[JOIN:.*]]
+; CHECK: [[T1]]:
+; CHECK-NEXT: br i1 [[D]], label %[[T1A:.*]], label %[[FLOW:.*]]
+; CHECK: [[T1A]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[T2]]:
+; CHECK-NEXT: br label %[[FLOW1]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[FLOW5:.*]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: br label %[[FLOW5]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW2:.*]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[FT]], label %[[FLOW5]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_FT]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ callbr void asm "", "!i"() to label %ft [label %ind]
+ft:
+ br i1 %c, label %t1, label %t2
+t1:
+ br i1 %d, label %t1a, label %join
+t1a:
+ br label %join
+t2:
+ br label %join
+join:
+ br label %exit
+ind:
+ br label %exit
+exit:
+ ret void
+}
+
+; a callbr fallthrough target contains a loop.
+define void @callbr_target_has_loop(i1 %c) {
+; CHECK-LABEL: define void @callbr_target_has_loop(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[FT:.*]] [label %[[IND:.*]]]
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[LOOP:.*]]:
+; CHECK-NEXT: br i1 [[C_INV]], label %[[FTEXIT:.*]], label %[[LOOP]]
+; CHECK: [[FLOW3:.*]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[FTEXIT]]:
+; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[LOOP]], label %[[FLOW3]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void asm "", "!i"() to label %ft [label %ind]
+ft:
+ br label %loop
+loop:
+ br i1 %c, label %loop, label %ftexit
+ftexit:
+ br label %exit
+ind:
+ br label %exit
+exit:
+ ret void
+}
+
+; one callbr target branches (conditionally) to another callbr target.
+define void @callbr_target_branches_to_target(i1 %d) {
+; CHECK-LABEL: define void @callbr_target_branches_to_target(
+; CHECK-SAME: i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[D_INV:%.*]] = xor i1 [[D]], true
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_T0:.*]] [label %[[ENTRY_TARGET_T1:.*]]]
+; CHECK: [[T0:.*]]:
+; CHECK-NEXT: br label %[[FLOW3:.*]]
+; CHECK: [[FLOW4:.*]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[T1:.*]] ], [ [[TMP1:%.*]], %[[FLOW3]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[T0A:.*]], label %[[EXIT:.*]]
+; CHECK: [[T0A]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: [[TMP1]] = phi i1 [ true, %[[T0]] ], [ false, %[[FLOW:.*]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ [[D_INV]], %[[T0]] ], [ true, %[[FLOW]] ]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[T1]], label %[[FLOW4]]
+; CHECK: [[T1]]:
+; CHECK-NEXT: br label %[[FLOW4]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_T0]] ], [ false, %[[ENTRY_TARGET_T1]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[T0]], label %[[FLOW3]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_T0]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[ENTRY_TARGET_T1]]:
+; CHECK-NEXT: br label %[[FLOW]]
+;
+entry:
+ callbr void asm "", "!i"() to label %t0 [label %t1]
+t0:
+ br i1 %d, label %t0a, label %t1
+t0a:
+ br label %exit
+t1:
+ br label %exit
+exit:
+ ret void
+}
+
+; a chain: each callbr target falls into the next one.
+define void @callbr_targets_chain(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_targets_chain(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[D_INV:%.*]] = xor i1 [[D]], true
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: callbr void asm "", "!i,!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_T0:.*]] [label %[[ENTRY_TARGET_T1:.*]], label %[[ENTRY_TARGET_T2:.*]]]
+; CHECK: [[FLOW5:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ true, %[[FLOW3:.*]] ], [ false, %[[FLOW:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ [[ISLAND_SEL1:%.*]], %[[FLOW3]] ], [ false, %[[FLOW]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[FLOW3]] ], [ true, %[[FLOW]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[T0:.*]], label %[[FLOW6:.*]]
+; CHECK: [[T0]]:
+; CHECK-NEXT: br label %[[FLOW6]]
+; CHECK: [[FLOW9:.*]]:
+; CHECK-NEXT: br i1 [[TMP5:%.*]], label %[[T0A:.*]], label %[[EXIT:.*]]
+; CHECK: [[T0A]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[FLOW6]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ true, %[[T0]] ], [ false, %[[FLOW5]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ [[C_INV]], %[[T0]] ], [ [[TMP1]], %[[FLOW5]] ]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[T1:.*]], label %[[FLOW7:.*]]
+; CHECK: [[T1]]:
+; CHECK-NEXT: br label %[[FLOW7]]
+; CHECK: [[FLOW8:.*]]:
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[T2:.*]] ], [ [[TMP8:%.*]], %[[FLOW7]] ]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[T1A:.*]], label %[[FLOW9]]
+; CHECK: [[T1A]]:
+; CHECK-NEXT: br label %[[FLOW9]]
+; CHECK: [[FLOW7]]:
+; CHECK-NEXT: [[TMP5]] = phi i1 [ false, %[[T1]] ], [ [[TMP6]], %[[FLOW6]] ]
+; CHECK-NEXT: [[TMP8]] = phi i1 [ true, %[[T1]] ], [ false, %[[FLOW6]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ [[D_INV]], %[[T1]] ], [ [[TMP0]], %[[FLOW6]] ]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[T2]], label %[[FLOW8]]
+; CHECK: [[T2]]:
+; CHECK-NEXT: br label %[[FLOW8]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_T0]] ], [ false, %[[ENTRY_TARGET_T1]] ], [ false, %[[ENTRY_TARGET_T2]] ]
+; CHECK-NEXT: [[ISLAND_SEL1]] = phi i1 [ false, %[[ENTRY_TARGET_T0]] ], [ true, %[[ENTRY_TARGET_T1]] ], [ false, %[[ENTRY_TARGET_T2]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW3]], label %[[FLOW5]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: br label %[[FLOW5]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_T0]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[ENTRY_TARGET_T1]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[ENTRY_TARGET_T2]]:
+; CHECK-NEXT: br label %[[FLOW]]
+;
+entry:
+ callbr void asm "", "!i,!i"() to label %t0 [label %t1, label %t2]
+t0:
+ br i1 %c, label %t0a, label %t1
+t0a:
+ br label %exit
+t1:
+ br i1 %d, label %t1a, label %t2
+t1a:
+ br label %exit
+t2:
+ br label %exit
+exit:
+ ret void
+}
+
+; ===== callbr as the region exit (reconvergence point) =====
+
+; callbr as the reconvergence (exit) of an if-diamond region
+define void @callbr_as_region_exit(i1 %c) {
+; CHECK-LABEL: define void @callbr_as_region_exit(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: br i1 [[C_INV]], label %[[B:.*]], label %[[FLOW2:.*]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[B]] ], [ true, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[A:.*]], label %[[MERGE:.*]]
+; CHECK: [[A]]:
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[B]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CONT:.*]] [label %[[IND:.*]]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CONT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %A, label %B
+A:
+ br label %merge
+B:
+ br label %merge
+merge:
+ callbr void asm "", "!i"() to label %cont [label %ind]
+cont:
+ br label %exit
+ind:
+ br label %exit
+exit:
+ ret void
+}
+
+; callbr exit that is also itself nested before another join
+define void @callbr_exit_then_more(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_exit_then_more(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: br i1 [[C_INV]], label %[[B:.*]], label %[[FLOW4:.*]]
+; CHECK: [[FLOW4]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[B]] ], [ true, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[A:.*]], label %[[MERGE:.*]]
+; CHECK: [[A]]:
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[B]]:
+; CHECK-NEXT: br label %[[FLOW4]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[MERGE_TARGET_CONT:.*]] [label %[[IND:.*]]]
+; CHECK: [[CONT:.*]]:
+; CHECK-NEXT: br i1 [[D]], label %[[T:.*]], label %[[FLOW:.*]]
+; CHECK: [[T]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW1:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[MERGE_TARGET_CONT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[CONT]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[MERGE_TARGET_CONT]]:
+; CHECK-NEXT: br label %[[FLOW1]]
+;
+entry:
+ br i1 %c, label %A, label %B
+A:
+ br label %merge
+B:
+ br label %merge
+merge:
+ callbr void asm "", "!i"() to label %cont [label %ind]
+cont:
+ br i1 %d, label %t, label %exit
+t:
+ br label %exit
+ind:
+ br label %exit
+exit:
+ ret void
+}
+
+; ===== Multiple / nested callbrs =====
+
+; A callbr destination is itself another callbr block (two callbr
+; terminators in one function): two nested islands.
+define void @callbr_target_is_callbr() {
+; CHECK-LABEL: define void @callbr_target_is_callbr() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_CB1:.*]] [label %[[IND0:.*]]]
+; CHECK: [[CB1:.*]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[FT1:.*]] [label %[[IND1:.*]]]
+; CHECK: [[FT1]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[IND1]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[IND0]]:
+; CHECK-NEXT: br label %[[FLOW2:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT1]] ], [ false, %[[IND1]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[FLOW3:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[FLOW3]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[ISLAND_SEL4:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_CB1]] ], [ false, %[[IND0]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL4]], label %[[CB1]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_CB1]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ callbr void asm "", "!i"() to label %cb1 [label %ind0]
+cb1:
+ callbr void asm "", "!i"() to label %ft1 [label %ind1]
+ft1:
+ br label %exit
+ind1:
+ br label %exit
+ind0:
+ br label %exit
+exit:
+ ret void
+}
+
+; ===== Data flow across the island (return value / phis) =====
+
+; A PHI in the join consumes the callbr return value across the bypass.
+define i32 @callbr_phi_uses_return_value(i1 %c) {
+; CHECK-LABEL: define i32 @callbr_phi_uses_return_value(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = callbr i32 asm "", "=r,!i"()
+; CHECK-NEXT: to label %[[FT:.*]] [label %[[IND:.*]]]
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[JOIN:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK: [[JOIN]]:
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[V]], %[[FLOW]] ], [ 0, %[[FLOW1]] ]
+; CHECK-NEXT: ret i32 [[P]]
+;
+entry:
+ %v = callbr i32 asm "", "=r,!i"() to label %ft [label %ind]
+ft:
+ br label %join
+ind:
+ br label %join
+join:
+ %p = phi i32 [ %v, %ft ], [ 0, %ind ]
+ ret i32 %p
+}
+
+; callbr return value used directly in the fallthrough block (not via a phi).
+define i32 @callbr_return_value_in_fallthrough() {
+; CHECK-LABEL: define i32 @callbr_return_value_in_fallthrough() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = callbr i32 asm "", "=r,!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_FT:.*]] [label %[[IND:.*]]]
+; CHECK: [[FT:.*]]:
+; CHECK-NEXT: [[W:%.*]] = add i32 [[V]], 1
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[FT]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i32 [ 0, %[[FLOW]] ], [ [[W]], %[[FT]] ]
+; CHECK-NEXT: ret i32 [[R]]
+; CHECK: [[ENTRY_TARGET_FT]]:
+; CHECK-NEXT: br label %[[FLOW]]
+;
+entry:
+ %v = callbr i32 asm "", "=r,!i"() to label %ft [label %ind]
+ft:
+ %w = add i32 %v, 1
+ br label %exit
+ind:
+ br label %exit
+exit:
+ %r = phi i32 [ %w, %ft ], [ 0, %ind ]
+ ret i32 %r
+}
+
+; ===== Loops =====
+
+; The callbr fallthrough edge loops back to the callbr block.
+define void @callbr_self_loop_fallthrough(i1 %c) {
+; CHECK-LABEL: define void @callbr_self_loop_fallthrough(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_CB:.*]] [label %[[IND:.*]]]
+; CHECK: [[FLOW2:.*]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_CB]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW3:.*]], label %[[CB]]
+; CHECK: [[FLOW3]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_CB]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ br label %cb
+cb:
+ callbr void asm "", "!i"() to label %cb [label %ind]
+ind:
+ br label %exit
+exit:
+ ret void
+}
+
+; The callbr indirect edge loops back to the callbr block.
+define void @callbr_self_loop_indirect(i1 %c) {
+; CHECK-LABEL: define void @callbr_self_loop_indirect(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[FT:.*]] [label %[[CB_TARGET_CB:.*]]]
+; CHECK: [[EXIT:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[CB_TARGET_CB]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[FT1:.*]], label %[[CB]]
+; CHECK: [[FT1]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[CB_TARGET_CB]]:
+; CHECK-NEXT: br label %[[EXIT]]
+;
+entry:
+ br label %cb
+cb:
+ callbr void asm "", "!i"() to label %ft [label %cb]
+ft:
+ br label %exit
+exit:
+ ret void
+}
+
+; The callbr fallthrough self-loops and the indirect edge goes straight to the
+; region's return block (no intermediate forwarder), so the loop's exit is a
+; direct callbr target rather than a Flow.
+define void @callbr_self_loop_exit_is_ret(i1 %c) {
+; CHECK-LABEL: define void @callbr_self_loop_exit_is_ret(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_CB:.*]] [label %[[CB_TARGET_RET:.*]]]
+; CHECK: [[FLOW:.*]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_CB]] ], [ false, %[[CB_TARGET_RET]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[RET:.*]], label %[[CB]]
+; CHECK: [[RET]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_CB]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[CB_TARGET_RET]]:
+; CHECK-NEXT: br label %[[FLOW]]
+;
+entry:
+ br label %cb
+cb:
+ callbr void asm "", "!i"() to label %cb [label %ret]
+ret:
+ ret void
+}
+
+; The callbr block is the header of a loop with a non-callbr latch.
+define void @callbr_as_loop_header(i1 %c) {
+; CHECK-LABEL: define void @callbr_as_loop_header(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[EXIT:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[TMP2:%.*]], %[[FLOW5:.*]] ], [ true, %[[FLOW2:.*]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW6:.*]], label %[[CB]]
+; CHECK: [[FLOW:.*]]:
+; CHECK-NEXT: br label %[[FLOW5]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_BODY:.*]] [label %[[CB_TARGET_EXIT:.*]]]
+; CHECK: [[FLOW3:.*]]:
+; CHECK-NEXT: br i1 [[C]], label %[[FLOW]], label %[[FLOW5]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ true, %[[CB_TARGET_BODY]] ], [ false, %[[CB_TARGET_EXIT]] ]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[FLOW3]], label %[[EXIT]]
+; CHECK: [[FLOW5]]:
+; CHECK-NEXT: [[TMP2]] = phi i1 [ false, %[[FLOW]] ], [ true, %[[FLOW3]] ]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[FLOW6]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_BODY]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[CB_TARGET_EXIT]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ br label %cb
+cb:
+ callbr void asm "", "!i"() to label %body [label %exit]
+body:
+ br i1 %c, label %cb, label %exit
+exit:
+ ret void
+}
+
+; A back edge whose target is a callbr that is also a region exit: the
+; if-diamond reconverges at %cb (callbr), and %cb's indirect target loops
+; back to %cb. This makes %cb both a region exit and a loop header.
+define void @callbr_exit_is_loop_header(i1 %c) {
+; CHECK-LABEL: define void @callbr_exit_is_loop_header(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: br i1 [[C_INV]], label %[[B:.*]], label %[[FLOW5:.*]]
+; CHECK: [[FLOW5]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[B]] ], [ true, %[[ENTRY]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[A:.*]], label %[[CB:.*]]
+; CHECK: [[A]]:
+; CHECK-NEXT: br label %[[CB]]
+; CHECK: [[B]]:
+; CHECK-NEXT: br label %[[FLOW5]]
+; CHECK: [[FLOW4:.*]]:
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[FLOW:.*]] ], [ true, %[[FLOW2:.*]] ]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[OUT:.*]], label %[[CB]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[FLOW4]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_OUT:.*]] [label %[[BODY:.*]]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[FLOW3:.*]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_OUT]] ], [ false, %[[BODY]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW3]], label %[[FLOW4]]
+; CHECK: [[OUT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_OUT]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ br i1 %c, label %A, label %B
+A:
+ br label %cb
+B:
+ br label %cb
+cb:
+ callbr void asm "", "!i"() to label %out [label %body]
+body:
+ br label %latch
+latch:
+ br label %cb
+out:
+ ret void
+}
+
+; The callbr is the LATCH of a loop whose header is a separate non-callbr block:
+; %header dominates %cb, so the callbr's fallthrough edge %cb -> %header is a
+; back edge, while the indirect edge %cb -> %out leaves the loop.
+define void @callbr_latch_separate_header(i1 %c) {
+; CHECK-LABEL: define void @callbr_latch_separate_header(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: br i1 [[C]], label %[[FLOW:.*]], label %[[FLOW4:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[FLOW4]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[CALLBR_SEL_INV:%.*]], %[[FLOW2:.*]] ], [ true, %[[HEADER]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW5:.*]], label %[[HEADER]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_HEADER:.*]] [label %[[OUT:.*]]]
+; CHECK: [[OUT]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_HEADER]] ], [ false, %[[OUT]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br label %[[FLOW4]]
+; CHECK: [[FLOW5]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_HEADER]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ br label %header
+header:
+ br i1 %c, label %cb, label %exit
+cb:
+ callbr void asm "", "!i"() to label %header [label %out]
+out:
+ br label %exit
+exit:
+ ret void
+}
+
+; The callbr sits inside a loop with a non-callbr header and latch.
+define void @callbr_inside_loop(i1 %c) {
+; CHECK-LABEL: define void @callbr_inside_loop(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: br i1 [[C]], label %[[CB:.*]], label %[[FLOW2:.*]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[FT:.*]] [label %[[IND:.*]]]
+; CHECK: [[FT]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[IND]]:
+; CHECK-NEXT: br label %[[FLOW]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[LATCH:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[C_INV]], %[[LATCH]] ], [ true, %[[HEADER]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %header
+header:
+ br i1 %c, label %cb, label %exit
+cb:
+ callbr void asm "", "!i"() to label %ft [label %ind]
+ft:
+ br label %latch
+ind:
+ br label %latch
+latch:
+ br i1 %c, label %header, label %exit
+exit:
+ ret void
+}
+
+; reducible loop among targets: only %t0 is a direct callbr target; %t0 and %body
+; form the loop, %t1 is the exit target.
+define void @callbr_target_starts_loop(i1 %c) {
+; CHECK-LABEL: define void @callbr_target_starts_loop(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[C_INV:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[ENTRY_TARGET_T0:.*]] [label %[[T1:.*]]]
+; CHECK: [[T0:.*]]:
+; CHECK-NEXT: br label %[[BODY:.*]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: br i1 [[C_INV]], label %[[FLOW:.*]], label %[[T0]]
+; CHECK: [[T1]]:
+; CHECK-NEXT: br label %[[FLOW1:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[FLOW1]]:
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_T0]] ], [ false, %[[T1]] ]
+; CHECK-NEXT: br i1 [[ISLAND_SEL]], label %[[T0]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[ENTRY_TARGET_T0]]:
+; CHECK-NEXT: br label %[[FLOW1]]
+;
+entry:
+ callbr void asm "", "!i"() to label %t0 [label %t1]
+t0:
+ br label %body
+body:
+ br i1 %c, label %t0, label %exit
+t1:
+ br label %exit
+exit:
+ ret void
+}
+
+; callbr is the latch of the INNER of two nested loops.
+define void @callbr_inner_latch_nested_loop(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_inner_latch_nested_loop(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[D_INV:%.*]] = xor i1 [[D]], true
+; CHECK-NEXT: br label %[[OUTER:.*]]
+; CHECK: [[OUTER]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: br i1 [[C]], label %[[FLOW:.*]], label %[[FLOW4:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[FLOW4]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[CALLBR_SEL_INV:%.*]], %[[FLOW2:.*]] ], [ true, %[[INNER]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW5:.*]], label %[[INNER]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_INNER:.*]] [label %[[CB_TARGET_INNER_EXIT:.*]]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_INNER]] ], [ false, %[[CB_TARGET_INNER_EXIT]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br label %[[FLOW4]]
+; CHECK: [[FLOW5]]:
+; CHECK-NEXT: br i1 [[D_INV]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_INNER]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[CB_TARGET_INNER_EXIT]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ br label %outer
+outer:
+ br label %inner
+inner:
+ br i1 %c, label %cb, label %inner.exit
+cb:
+ callbr void asm "", "!i"() to label %inner [label %inner.exit]
+inner.exit:
+ br i1 %d, label %outer, label %exit
+exit:
+ ret void
+}
+
+; callbr with TWO back-edge successors: one to the inner header, one to the
+; outer header (break-to-outer-loop).
+define void @callbr_two_backedges(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_two_backedges(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[D_INV:%.*]] = xor i1 [[D]], true
+; CHECK-NEXT: br label %[[OUTER:.*]]
+; CHECK: [[OUTER]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: br i1 [[C]], label %[[FLOW:.*]], label %[[FLOW4:.*]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[CB:.*]]
+; CHECK: [[FLOW4]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[CALLBR_SEL:%.*]], %[[FLOW2:.*]] ], [ true, %[[INNER]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[CALLBR_SEL_INV:%.*]], %[[FLOW2]] ], [ true, %[[INNER]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW5:.*]], label %[[INNER]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[CB_TARGET_INNER:.*]] [label %[[CB_TARGET_OUTER:.*]]]
+; CHECK: [[FLOW2]]:
+; CHECK-NEXT: [[CALLBR_SEL]] = phi i1 [ true, %[[CB_TARGET_INNER]] ], [ false, %[[CB_TARGET_OUTER]] ]
+; CHECK-NEXT: [[CALLBR_SEL_INV]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: br label %[[FLOW4]]
+; CHECK: [[FLOW5]]:
+; CHECK-NEXT: br i1 [[TMP2]], label %[[INNER_EXIT:.*]], label %[[OUTER]]
+; CHECK: [[INNER_EXIT]]:
+; CHECK-NEXT: br i1 [[D_INV]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[CB_TARGET_INNER]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+; CHECK: [[CB_TARGET_OUTER]]:
+; CHECK-NEXT: br label %[[FLOW2]]
+;
+entry:
+ br label %outer
+outer:
+ br label %inner
+inner:
+ br i1 %c, label %cb, label %inner.exit
+cb:
+ callbr void asm "", "!i"() to label %inner [label %outer]
+inner.exit:
+ br i1 %d, label %outer, label %exit
+exit:
+ ret void
+}
+
+; intrinsic callbr (kill) that is the latch of a loop.
+define void @callbr_kill_in_loop(i1 %c, i1 %d) {
+; CHECK-LABEL: define void @callbr_kill_in_loop(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: br i1 [[C]], label %[[CB:.*]], label %[[EXIT:.*]]
+; CHECK: [[CB]]:
+; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[D]])
+; CHECK-NEXT: to label %[[HEADER]] [label %[[DEAD:.*]]]
+; CHECK: [[DEAD]]:
+; CHECK-NEXT: unreachable
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %header
+header:
+ br i1 %c, label %cb, label %exit
+cb:
+ callbr void @llvm.amdgcn.kill(i1 %d) to label %header [label %dead]
+dead:
+ unreachable
+exit:
+ ret void
+}
>From 91d54b49be748fdc700393b6a99c5fdc33be555a Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 8 Jun 2026 08:21:36 -0500
Subject: [PATCH 2/2] make island naming in tests consistent; improve two edge
cases
---
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 30 ++-
llvm/test/Transforms/StructurizeCFG/callbr.ll | 198 ++++++++++++------
2 files changed, 143 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index d1ada89d9d246..93ff98679cab4 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -701,14 +701,6 @@ void StructurizeCFG::gatherPredicates(RegionNode *N) {
LPred[P] = buildCondition(CondBr, Idx, true);
}
}
- // } else if (isIsland(*P)) {
- // // Island edges have no SSA condition, so we treat them as an
- // // unconditional branch.
- // if (Visited.count(P))
- // Pred[P] = {BoolTrue, std::nullopt};
- // else
- // LPred[P] = {BoolFalse, std::nullopt};
- // }
} else {
// It's an exit from a sub region
while (R->getParent() != ParentRegion)
@@ -1279,13 +1271,13 @@ void StructurizeCFG::wireFlow(bool ExitUseAllowed,
return;
}
- // A callbr forwarder, a simple single-branch block reached straight from the
- // callbr (created by handleIsland's edge split, or a reused trivial target),
- // has already been converged into the dispatch ladder. Close the tail and
- // leave it, so the normal machinery does not redirect it out of the ladder.
+ // Leave an island's own targets alone: a forwarder already converged into the
+ // dispatch ladder, or an unreachable target kept as a direct dead lane. Close
+ // the tail so the normal machinery does not redirect or structurize either.
bool HasIslandPred = llvm::any_of(
predecessors(Entry), [](BasicBlock *Pred) { return isIsland(*Pred); });
- if (HasIslandPred && isIntermediateTarget(*Entry)) {
+ if (HasIslandPred &&
+ (isIntermediateTarget(*Entry) || isUnreachableTarget(*Entry))) {
Tail.close(Node);
return;
}
@@ -1406,11 +1398,15 @@ void StructurizeCFG::handleIsland(BasicBlock *IslandBB) {
// dominates the whole ladder, so the sel phis are usable at every level. If
// the island has a self-loop, it is dispatched by the ladder as well.
//
- // Edge edits are recorded in DTUpdates and applied via the same
- // DomTreeUpdater, keeping the DominatorTree incrementally correct. The island
- // Flow blocks are created without DT->addNewBlock and get their idoms from
- // the inserted edges, so we must not mix in a direct getNextFlow here.
+ // Unlike getNextFlow, MakeFlow does not register its block in the
+ // DominatorTree (no DT->addNewBlock). Instead the edits below are collected
+ // in DTUpdates and applied in one pass, and the DomTreeUpdater derives each
+ // new Flow's idom from the inserted edges. So every MakeFlow block must be
+ // the target of at least one Insert before applyUpdates, which holds by
+ // construction below.
unsigned N = Targets.size();
+ if (!N)
+ return; // All callbr targets are unreachable; nothing to dispatch.
SmallVector<DominatorTree::UpdateType, 16> DTUpdates;
auto MakeFlow = [&]() {
BasicBlock *Flow = BasicBlock::Create(Func->getContext(), FlowBlockName,
diff --git a/llvm/test/Transforms/StructurizeCFG/callbr.ll b/llvm/test/Transforms/StructurizeCFG/callbr.ll
index acf2f6e254148..ac93d0b6347c0 100644
--- a/llvm/test/Transforms/StructurizeCFG/callbr.ll
+++ b/llvm/test/Transforms/StructurizeCFG/callbr.ll
@@ -7,7 +7,7 @@
; unconditional branch as the only instruction) is interposed with a bypass
; block (via SplitCallBrEdge) so the targets are not direct successors of the
; immutable callbr; the bypasses then converge into a Flow block whose
-; "callbr.sel" phis recover, as data, which target the callbr selected, and a
+; "island.sel" phis recover, as data, which target the callbr selected, and a
; ladder of 2-way Flow branches re-dispatches to the targets. Loop back-edges
; among the targets are routed so the normal loop machinery closes the loop. The
; callbr terminator itself is never rewritten.
@@ -27,9 +27,9 @@ define void @callbr_simple() {
; CHECK: [[INDIRECT]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -82,25 +82,25 @@ define void @callbr_many_indirect(i1 %c) {
; CHECK: [[I3]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[I0]] ], [ false, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
-; CHECK-NEXT: [[CALLBR_SEL1:%.*]] = phi i1 [ false, %[[FT]] ], [ true, %[[I0]] ], [ false, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
-; CHECK-NEXT: [[CALLBR_SEL2:%.*]] = phi i1 [ false, %[[FT]] ], [ false, %[[I0]] ], [ true, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
-; CHECK-NEXT: [[CALLBR_SEL3:%.*]] = phi i1 [ false, %[[FT]] ], [ false, %[[I0]] ], [ false, %[[I1]] ], [ true, %[[I2]] ], [ false, %[[I3]] ]
-; CHECK-NEXT: [[CALLBR_SEL3_INV:%.*]] = xor i1 [[CALLBR_SEL3]], true
-; CHECK-NEXT: [[CALLBR_SEL2_INV:%.*]] = xor i1 [[CALLBR_SEL2]], true
-; CHECK-NEXT: [[CALLBR_SEL1_INV:%.*]] = xor i1 [[CALLBR_SEL1]], true
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW4:.*]], label %[[FLOW8:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[I0]] ], [ false, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[ISLAND_SEL1:%.*]] = phi i1 [ false, %[[FT]] ], [ true, %[[I0]] ], [ false, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[ISLAND_SEL2:%.*]] = phi i1 [ false, %[[FT]] ], [ false, %[[I0]] ], [ true, %[[I1]] ], [ false, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[ISLAND_SEL3:%.*]] = phi i1 [ false, %[[FT]] ], [ false, %[[I0]] ], [ false, %[[I1]] ], [ true, %[[I2]] ], [ false, %[[I3]] ]
+; CHECK-NEXT: [[ISLAND_SEL3_INV:%.*]] = xor i1 [[ISLAND_SEL3]], true
+; CHECK-NEXT: [[ISLAND_SEL2_INV:%.*]] = xor i1 [[ISLAND_SEL2]], true
+; CHECK-NEXT: [[ISLAND_SEL1_INV:%.*]] = xor i1 [[ISLAND_SEL1]], true
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW4:.*]], label %[[FLOW8:.*]]
; CHECK: [[FLOW4]]:
-; CHECK-NEXT: br i1 [[CALLBR_SEL1_INV]], label %[[FLOW5:.*]], label %[[FLOW9:.*]]
+; CHECK-NEXT: br i1 [[ISLAND_SEL1_INV]], label %[[FLOW5:.*]], label %[[FLOW9:.*]]
; CHECK: [[FLOW8]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[FLOW5]]:
-; CHECK-NEXT: br i1 [[CALLBR_SEL2_INV]], label %[[FLOW6:.*]], label %[[FLOW10:.*]]
+; CHECK-NEXT: br i1 [[ISLAND_SEL2_INV]], label %[[FLOW6:.*]], label %[[FLOW10:.*]]
; CHECK: [[FLOW9]]:
; CHECK-NEXT: br label %[[FLOW8]]
; CHECK: [[FLOW6]]:
-; CHECK-NEXT: br i1 [[CALLBR_SEL3_INV]], label %[[FLOW7:.*]], label %[[FLOW11:.*]]
+; CHECK-NEXT: br i1 [[ISLAND_SEL3_INV]], label %[[FLOW7:.*]], label %[[FLOW11:.*]]
; CHECK: [[FLOW10]]:
; CHECK-NEXT: br label %[[FLOW9]]
; CHECK: [[FLOW7]]:
@@ -133,9 +133,9 @@ define void @callbr_shared_target() {
; CHECK-NEXT: callbr void asm "", "!i"()
; CHECK-NEXT: to label %[[ENTRY_TARGET_FT:.*]] [label %[[ENTRY_TARGET_FT1:.*]]]
; CHECK: [[FLOW:.*]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FT]] ], [ false, %[[ENTRY_TARGET_FT1]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW2:.*]], label %[[FT:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FT]] ], [ false, %[[ENTRY_TARGET_FT1]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW2:.*]], label %[[FT:.*]]
; CHECK: [[FLOW2]]:
; CHECK-NEXT: br label %[[FT]]
; CHECK: [[FT]]:
@@ -184,10 +184,10 @@ define void @callbr_multi_indirect_one_trivial(i1 %c, i1 %d) {
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[FLOW8]]
; CHECK: [[FLOW2]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FALLTHROUGH]] ], [ false, %[[ENTRY_TARGET_INDIRECT0]] ], [ false, %[[INDIRECT1]] ]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_FALLTHROUGH]] ], [ false, %[[ENTRY_TARGET_INDIRECT0]] ], [ false, %[[INDIRECT1]] ]
; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[ENTRY_TARGET_FALLTHROUGH]] ], [ true, %[[ENTRY_TARGET_INDIRECT0]] ], [ false, %[[INDIRECT1]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW4:.*]], label %[[FLOW6]]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW4:.*]], label %[[FLOW6]]
; CHECK: [[FLOW4]]:
; CHECK-NEXT: br i1 [[TMP1]], label %[[FALLTHROUGH]], label %[[EXIT]]
; CHECK: [[FLOW8]]:
@@ -287,6 +287,68 @@ exit:
ret void
}
+; Two unreachable targets: both stay direct dead lanes (out of the ladder) and
+; reconverge at the exit, giving it three predecessors.
+define void @callbr_two_unreachable_targets() {
+; CHECK-LABEL: define void @callbr_two_unreachable_targets() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", "!i,!i"()
+; CHECK-NEXT: to label %[[LIVE:.*]] [label %[[DEAD1:.*]], label %[[DEAD2:.*]]]
+; CHECK: [[LIVE]]:
+; CHECK-NEXT: br label %[[FLOW:.*]]
+; CHECK: [[DEAD1]]:
+; CHECK-NEXT: call void @llvm.amdgcn.unreachable()
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[DEAD2]]:
+; CHECK-NEXT: call void @llvm.amdgcn.unreachable()
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[FLOW]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void asm "", "!i,!i"() to label %live [label %dead1, label %dead2]
+live:
+ br label %exit
+dead1:
+ call void @llvm.amdgcn.unreachable()
+ br label %exit
+dead2:
+ call void @llvm.amdgcn.unreachable()
+ br label %exit
+exit:
+ ret void
+}
+
+; Every target is unreachable: there is nothing to dispatch, so no ladder is
+; built and the callbr is left untouched with both dead lanes direct.
+define void @callbr_all_targets_unreachable() {
+; CHECK-LABEL: define void @callbr_all_targets_unreachable() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: callbr void asm "", "!i"()
+; CHECK-NEXT: to label %[[DEAD0:.*]] [label %[[DEAD1:.*]]]
+; CHECK: [[DEAD0]]:
+; CHECK-NEXT: call void @llvm.amdgcn.unreachable()
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[DEAD1]]:
+; CHECK-NEXT: call void @llvm.amdgcn.unreachable()
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ callbr void asm "", "!i"() to label %dead0 [label %dead1]
+dead0:
+ call void @llvm.amdgcn.unreachable()
+ br label %exit
+dead1:
+ call void @llvm.amdgcn.unreachable()
+ br label %exit
+exit:
+ ret void
+}
+
; ===== callbr nested in / surrounded by non-callbr control flow =====
; Callbr nested in non-callbr: non-callbr is transformed
@@ -308,9 +370,9 @@ define void @callbr_in_non_callbr(i1 %c) {
; CHECK: [[NOCALLBR]]:
; CHECK-NEXT: br label %[[FLOW2]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[FLOW3:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FALLTHROUGH]] ], [ false, %[[INDIRECT]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[FLOW3:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[FLOW3]]
; CHECK: [[FLOW3]]:
@@ -407,9 +469,9 @@ define void @callbr_nested_in_non_callbr(i1 %c, i1 %d, i1 %e, i1 %f) {
; CHECK: [[FLOW2]]:
; CHECK-NEXT: br label %[[FLOW6]]
; CHECK: [[FLOW3]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CALLBR_TARGET_FALLTHROUGH]] ], [ false, %[[CALLBR_TARGET_INDIRECT]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW4]], label %[[FLOW5]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CALLBR_TARGET_FALLTHROUGH]] ], [ false, %[[CALLBR_TARGET_INDIRECT]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW4]], label %[[FLOW5]]
; CHECK: [[FLOW6]]:
; CHECK-NEXT: br label %[[RET]]
; CHECK: [[RET]]:
@@ -461,9 +523,9 @@ define void @callbr_regions_around(i1 %c, i1 %d, i1 %e) {
; CHECK: [[IND]]:
; CHECK-NEXT: br label %[[FLOW1]]
; CHECK: [[FLOW1]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW2:.*]], label %[[POST:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW2:.*]], label %[[POST:.*]]
; CHECK: [[FLOW2]]:
; CHECK-NEXT: br label %[[POST]]
; CHECK: [[POST]]:
@@ -526,9 +588,9 @@ define void @callbr_pred_from_subregion(i1 %c, i1 %d) {
; CHECK: [[IND]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CONT]] ], [ false, %[[IND]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CONT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -727,10 +789,10 @@ define void @callbr_targets_chain(i1 %c, i1 %d) {
; CHECK: [[T2]]:
; CHECK-NEXT: br label %[[FLOW8]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_T0]] ], [ false, %[[ENTRY_TARGET_T1]] ], [ false, %[[ENTRY_TARGET_T2]] ]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[ENTRY_TARGET_T0]] ], [ false, %[[ENTRY_TARGET_T1]] ], [ false, %[[ENTRY_TARGET_T2]] ]
; CHECK-NEXT: [[ISLAND_SEL1]] = phi i1 [ false, %[[ENTRY_TARGET_T0]] ], [ true, %[[ENTRY_TARGET_T1]] ], [ false, %[[ENTRY_TARGET_T2]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW3]], label %[[FLOW5]]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW3]], label %[[FLOW5]]
; CHECK: [[FLOW3]]:
; CHECK-NEXT: br label %[[FLOW5]]
; CHECK: [[EXIT]]:
@@ -782,9 +844,9 @@ define void @callbr_as_region_exit(i1 %c) {
; CHECK: [[IND]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CONT]] ], [ false, %[[IND]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CONT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[EXIT:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -876,9 +938,9 @@ define void @callbr_target_is_callbr() {
; CHECK: [[IND0]]:
; CHECK-NEXT: br label %[[FLOW2:.*]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT1]] ], [ false, %[[IND1]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[FLOW3:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FT1]] ], [ false, %[[IND1]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[FLOW3:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[FLOW3]]
; CHECK: [[FLOW3]]:
@@ -919,9 +981,9 @@ define i32 @callbr_phi_uses_return_value(i1 %c) {
; CHECK: [[IND]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[JOIN:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[JOIN:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[JOIN]]
; CHECK: [[JOIN]]:
@@ -983,9 +1045,9 @@ define void @callbr_self_loop_fallthrough(i1 %c) {
; CHECK-NEXT: callbr void asm "", "!i"()
; CHECK-NEXT: to label %[[CB_TARGET_CB:.*]] [label %[[IND:.*]]]
; CHECK: [[FLOW2:.*]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_CB]] ], [ false, %[[IND]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW3:.*]], label %[[CB]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_CB]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW3:.*]], label %[[CB]]
; CHECK: [[FLOW3]]:
; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[EXIT1]]:
@@ -1048,9 +1110,9 @@ define void @callbr_self_loop_exit_is_ret(i1 %c) {
; CHECK-NEXT: callbr void asm "", "!i"()
; CHECK-NEXT: to label %[[CB_TARGET_CB:.*]] [label %[[CB_TARGET_RET:.*]]]
; CHECK: [[FLOW:.*]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_CB]] ], [ false, %[[CB_TARGET_RET]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[RET:.*]], label %[[CB]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_CB]] ], [ false, %[[CB_TARGET_RET]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[RET:.*]], label %[[CB]]
; CHECK: [[RET]]:
; CHECK-NEXT: ret void
; CHECK: [[CB_TARGET_CB]]:
@@ -1134,9 +1196,9 @@ define void @callbr_exit_is_loop_header(i1 %c) {
; CHECK: [[FLOW3:.*]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW2]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_OUT]] ], [ false, %[[BODY]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW3]], label %[[FLOW4]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_OUT]] ], [ false, %[[BODY]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW3]], label %[[FLOW4]]
; CHECK: [[OUT]]:
; CHECK-NEXT: ret void
; CHECK: [[CB_TARGET_OUT]]:
@@ -1171,7 +1233,7 @@ define void @callbr_latch_separate_header(i1 %c) {
; CHECK: [[FLOW]]:
; CHECK-NEXT: br label %[[CB:.*]]
; CHECK: [[FLOW4]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[CALLBR_SEL_INV:%.*]], %[[FLOW2:.*]] ], [ true, %[[HEADER]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[ISLAND_SEL_INV:%.*]], %[[FLOW2:.*]] ], [ true, %[[HEADER]] ]
; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW5:.*]], label %[[HEADER]]
; CHECK: [[CB]]:
; CHECK-NEXT: callbr void asm "", "!i"()
@@ -1179,8 +1241,8 @@ define void @callbr_latch_separate_header(i1 %c) {
; CHECK: [[OUT]]:
; CHECK-NEXT: br label %[[FLOW2]]
; CHECK: [[FLOW2]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_HEADER]] ], [ false, %[[OUT]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_HEADER]] ], [ false, %[[OUT]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV]] = xor i1 [[ISLAND_SEL]], true
; CHECK-NEXT: br label %[[FLOW4]]
; CHECK: [[FLOW5]]:
; CHECK-NEXT: ret void
@@ -1216,9 +1278,9 @@ define void @callbr_inside_loop(i1 %c) {
; CHECK: [[IND]]:
; CHECK-NEXT: br label %[[FLOW]]
; CHECK: [[FLOW]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV:%.*]] = xor i1 [[CALLBR_SEL]], true
-; CHECK-NEXT: br i1 [[CALLBR_SEL_INV]], label %[[FLOW1:.*]], label %[[LATCH:.*]]
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[FT]] ], [ false, %[[IND]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV:%.*]] = xor i1 [[ISLAND_SEL]], true
+; CHECK-NEXT: br i1 [[ISLAND_SEL_INV]], label %[[FLOW1:.*]], label %[[LATCH:.*]]
; CHECK: [[FLOW1]]:
; CHECK-NEXT: br label %[[LATCH]]
; CHECK: [[FLOW2]]:
@@ -1296,14 +1358,14 @@ define void @callbr_inner_latch_nested_loop(i1 %c, i1 %d) {
; CHECK: [[FLOW]]:
; CHECK-NEXT: br label %[[CB:.*]]
; CHECK: [[FLOW4]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[CALLBR_SEL_INV:%.*]], %[[FLOW2:.*]] ], [ true, %[[INNER]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[ISLAND_SEL_INV:%.*]], %[[FLOW2:.*]] ], [ true, %[[INNER]] ]
; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW5:.*]], label %[[INNER]]
; CHECK: [[CB]]:
; CHECK-NEXT: callbr void asm "", "!i"()
; CHECK-NEXT: to label %[[CB_TARGET_INNER:.*]] [label %[[CB_TARGET_INNER_EXIT:.*]]]
; CHECK: [[FLOW2]]:
-; CHECK-NEXT: [[CALLBR_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_INNER]] ], [ false, %[[CB_TARGET_INNER_EXIT]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: [[ISLAND_SEL:%.*]] = phi i1 [ true, %[[CB_TARGET_INNER]] ], [ false, %[[CB_TARGET_INNER_EXIT]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV]] = xor i1 [[ISLAND_SEL]], true
; CHECK-NEXT: br label %[[FLOW4]]
; CHECK: [[FLOW5]]:
; CHECK-NEXT: br i1 [[D_INV]], label %[[EXIT:.*]], label %[[OUTER]]
@@ -1343,15 +1405,15 @@ define void @callbr_two_backedges(i1 %c, i1 %d) {
; CHECK: [[FLOW]]:
; CHECK-NEXT: br label %[[CB:.*]]
; CHECK: [[FLOW4]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[CALLBR_SEL:%.*]], %[[FLOW2:.*]] ], [ true, %[[INNER]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[CALLBR_SEL_INV:%.*]], %[[FLOW2]] ], [ true, %[[INNER]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[ISLAND_SEL:%.*]], %[[FLOW2:.*]] ], [ true, %[[INNER]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[ISLAND_SEL_INV:%.*]], %[[FLOW2]] ], [ true, %[[INNER]] ]
; CHECK-NEXT: br i1 [[TMP0]], label %[[FLOW5:.*]], label %[[INNER]]
; CHECK: [[CB]]:
; CHECK-NEXT: callbr void asm "", "!i"()
; CHECK-NEXT: to label %[[CB_TARGET_INNER:.*]] [label %[[CB_TARGET_OUTER:.*]]]
; CHECK: [[FLOW2]]:
-; CHECK-NEXT: [[CALLBR_SEL]] = phi i1 [ true, %[[CB_TARGET_INNER]] ], [ false, %[[CB_TARGET_OUTER]] ]
-; CHECK-NEXT: [[CALLBR_SEL_INV]] = xor i1 [[CALLBR_SEL]], true
+; CHECK-NEXT: [[ISLAND_SEL]] = phi i1 [ true, %[[CB_TARGET_INNER]] ], [ false, %[[CB_TARGET_OUTER]] ]
+; CHECK-NEXT: [[ISLAND_SEL_INV]] = xor i1 [[ISLAND_SEL]], true
; CHECK-NEXT: br label %[[FLOW4]]
; CHECK: [[FLOW5]]:
; CHECK-NEXT: br i1 [[TMP2]], label %[[INNER_EXIT:.*]], label %[[OUTER]]
More information about the llvm-commits
mailing list