[llvm] 3cbbded - Introduce unify-loop-exits pass.
Sameer Sahasrabuddhe via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 30 10:24:46 PDT 2020
Author: Sameer Sahasrabuddhe
Date: 2020-03-30T13:23:56-04:00
New Revision: 3cbbded68c260ea5574798ca444b3feecfbbcd70
URL: https://github.com/llvm/llvm-project/commit/3cbbded68c260ea5574798ca444b3feecfbbcd70
DIFF: https://github.com/llvm/llvm-project/commit/3cbbded68c260ea5574798ca444b3feecfbbcd70.diff
LOG: Introduce unify-loop-exits pass.
For each natural loop with multiple exit blocks, this pass creates a
new block N such that all exiting blocks now branch to N, and then
control flow is redistributed to all the original exit blocks.
The bulk of the tranformation is a new function introduced in
BasicBlockUtils that an redirect control flow from a set of incoming
blocks to a set of outgoing blocks via a common "hub".
This is a useful workaround for a limitation in the structurizer which
incorrectly orders blocks when processing a nest of loops. This pass
bypasses that issue by ensuring that each natural loop is recognized
as a separate region. Since the structurizer is a region pass, it no
longer sees a nest of loops in a single region, and instead processes
each "level" in the nesting as a separate region.
The AMDGPU backend provides a new option to enable this pass before
the structurizer, which may eventually be enabled by default.
Reviewers: madhur13490, arsenm, nhaehnle
Reviewed By: nhaehnle
Differential Revision: https://reviews.llvm.org/D75865
Added:
llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
llvm/test/Transforms/UnifyLoopExits/basic.ll
llvm/test/Transforms/UnifyLoopExits/nested.ll
llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
llvm/test/Transforms/UnifyLoopExits/switch.ll
Modified:
llvm/include/llvm/InitializePasses.h
llvm/include/llvm/LinkAllPasses.h
llvm/include/llvm/Transforms/Utils.h
llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
llvm/include/llvm/Transforms/Utils/Local.h
llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
llvm/lib/Transforms/Utils/CMakeLists.txt
llvm/lib/Transforms/Utils/Local.cpp
llvm/lib/Transforms/Utils/Utils.cpp
llvm/test/CodeGen/AMDGPU/loop_break.ll
llvm/tools/llc/llc.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index b3bbf5ee26e6..263fa87f6b75 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -415,6 +415,7 @@ void initializeTwoAddressInstructionPassPass(PassRegistry&);
void initializeTypeBasedAAWrapperPassPass(PassRegistry&);
void initializeTypePromotionPass(PassRegistry&);
void initializeUnifyFunctionExitNodesPass(PassRegistry&);
+void initializeUnifyLoopExitsPass(PassRegistry &);
void initializeUnpackMachineBundlesPass(PassRegistry&);
void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&);
void initializeUnreachableMachineBlockElimPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 51d89c4b1601..d5e9ffc1f80c 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -229,7 +229,8 @@ namespace {
(void) llvm::createScalarizeMaskedMemIntrinPass();
(void) llvm::createWarnMissedTransformationsPass();
(void) llvm::createHardwareLoopsPass();
- (void)llvm::createInjectTLIMappingsLegacyPass();
+ (void) llvm::createInjectTLIMappingsLegacyPass();
+ (void) llvm::createUnifyLoopExitsPass();
(void)new llvm::IntervalPartition();
(void)new llvm::ScalarEvolutionWrapperPass();
diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h
index bb31646ce462..402caf7a7d3d 100644
--- a/llvm/include/llvm/Transforms/Utils.h
+++ b/llvm/include/llvm/Transforms/Utils.h
@@ -126,6 +126,14 @@ FunctionPass *createControlHeightReductionLegacyPass();
// scalar-to-vector mappings from the TargetLibraryInfo.
//
FunctionPass *createInjectTLIMappingsLegacyPass();
+
+//===----------------------------------------------------------------------===//
+//
+// UnifyLoopExits - For each loop, creates a new block N such that all exiting
+// blocks branch to N, and then N distributes control flow to all the original
+// exit blocks.
+//
+FunctionPass *createUnifyLoopExitsPass();
}
#endif
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index f87bee5a6c83..905233b54c9c 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -17,6 +17,7 @@
// FIXME: Move to this file: BasicBlock::removePredecessor, BB::splitBasicBlock
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
@@ -363,6 +364,81 @@ bool SplitIndirectBrCriticalEdges(Function &F,
BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFI = nullptr);
+/// Given a set of incoming and outgoing blocks, create a "hub" such that every
+/// edge from an incoming block InBB to an outgoing block OutBB is now split
+/// into two edges, one from InBB to the hub and another from the hub to
+/// OutBB. The hub consists of a series of guard blocks, one for each outgoing
+/// block. Each guard block conditionally branches to the corresponding outgoing
+/// block, or the next guard block in the chain. These guard blocks are returned
+/// in the argument vector.
+///
+/// Since the control flow edges from InBB to OutBB have now been replaced, the
+/// function also updates any PHINodes in OutBB. For each such PHINode, the
+/// operands corresponding to incoming blocks are moved to a new PHINode in the
+/// hub, and the hub is made an operand of the original PHINode.
+///
+/// Input CFG:
+/// ----------
+///
+/// Def
+/// |
+/// v
+/// In1 In2
+/// | |
+/// | |
+/// v v
+/// Foo ---> Out1 Out2
+/// |
+/// v
+/// Use
+///
+///
+/// Create hub: Incoming = {In1, In2}, Outgoing = {Out1, Out2}
+/// ----------------------------------------------------------
+///
+/// Def
+/// |
+/// v
+/// In1 In2 Foo
+/// | Hub | |
+/// | + - - | - - + |
+/// | ' v ' V
+/// +------> Guard1 -----> Out1
+/// ' | '
+/// ' v '
+/// ' Guard2 -----> Out2
+/// ' ' |
+/// + - - - - - + |
+/// v
+/// Use
+///
+/// Limitations:
+/// -----------
+/// 1. This assumes that all terminators in the CFG are direct branches (the
+/// "br" instruction). The presence of any other control flow such as
+/// indirectbr, switch or callbr will cause an assert.
+///
+/// 2. The updates to the PHINodes are not sufficient to restore SSA
+/// form. Consider a definition Def, its use Use, incoming block In2 and
+/// outgoing block Out2, such that:
+/// a. In2 is reachable from D or contains D.
+/// b. U is reachable from Out2 or is contained in Out2.
+/// c. U is not a PHINode if U is contained in Out2.
+///
+/// Clearly, Def dominates Out2 since the program is valid SSA. But when the
+/// hub is introduced, there is a new path through the hub along which Use is
+/// reachable from entry without passing through Def, and SSA is no longer
+/// valid. To fix this, we need to look at all the blocks post-dominated by
+/// the hub on the one hand, and dominated by Out2 on the other. This is left
+/// for the caller to accomplish, since each specific use of this function
+/// may have additional information which simplifies this fixup. For example,
+/// see restoreSSA() in the UnifyLoopExits pass.
+BasicBlock *CreateControlFlowHub(DomTreeUpdater *DTU,
+ SmallVectorImpl<BasicBlock *> &GuardBlocks,
+ const SetVector<BasicBlock *> &Predecessors,
+ const SetVector<BasicBlock *> &Successors,
+ const StringRef Prefix);
+
} // end namespace llvm
#endif // LLVM_TRANSFORMS_UTILS_BASICBLOCKUTILS_H
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 4c717100a402..c8afb5c3f304 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -533,6 +533,13 @@ void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI,
/// value?
bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx);
+//===----------------------------------------------------------------------===//
+// Value helper functions
+//
+
+/// Invert the given true/false value, possibly reusing an existing copy.
+Value *invertCondition(Value *Condition);
+
} // end namespace llvm
#endif // LLVM_TRANSFORMS_UTILS_LOCAL_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 6908f43b4dbd..17c48a96cd5e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -192,6 +192,11 @@ static cl::opt<bool> EnableScalarIRPasses(
cl::init(true),
cl::Hidden);
+static cl::opt<bool> EnableStructurizerWorkarounds(
+ "amdgpu-enable-structurizer-workarounds",
+ cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(false),
+ cl::Hidden);
+
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
// Register the target
RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
@@ -858,6 +863,9 @@ bool GCNPassConfig::addPreISel() {
// regions formed by them.
addPass(&AMDGPUUnifyDivergentExitNodesID);
if (!LateCFGStructurize) {
+ if (EnableStructurizerWorkarounds) {
+ addPass(createUnifyLoopExitsPass());
+ }
addPass(createStructurizeCFGPass(true)); // true -> SkipUniformRegions
}
addPass(createSinkingPass());
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 15aad801ecc9..84d8eebd9720 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -44,6 +44,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#include <algorithm>
#include <cassert>
@@ -218,8 +219,6 @@ class StructurizeCFG : public RegionPass {
void analyzeLoops(RegionNode *N);
- Value *invert(Value *Condition);
-
Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
void gatherPredicates(RegionNode *N);
@@ -405,39 +404,6 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) {
}
}
-/// Invert the given condition
-Value *StructurizeCFG::invert(Value *Condition) {
- // First: Check if it's a constant
- if (Constant *C = dyn_cast<Constant>(Condition))
- return ConstantExpr::getNot(C);
-
- // Second: If the condition is already inverted, return the original value
- Value *NotCondition;
- if (match(Condition, m_Not(m_Value(NotCondition))))
- return NotCondition;
-
- if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
- // Third: Check all the users for an invert
- BasicBlock *Parent = Inst->getParent();
- for (User *U : Condition->users())
- if (Instruction *I = dyn_cast<Instruction>(U))
- if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
- return I;
-
- // Last option: Create a new instruction
- return BinaryOperator::CreateNot(Condition, "", Parent->getTerminator());
- }
-
- if (Argument *Arg = dyn_cast<Argument>(Condition)) {
- BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
- return BinaryOperator::CreateNot(Condition,
- Arg->getName() + ".inv",
- EntryBlock.getTerminator());
- }
-
- llvm_unreachable("Unhandled condition to invert");
-}
-
/// Build the condition for one edge
Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
bool Invert) {
@@ -446,7 +412,7 @@ Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
Cond = Term->getCondition();
if (Idx != (unsigned)Invert)
- Cond = invert(Cond);
+ Cond = invertCondition(Cond);
}
return Cond;
}
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 2fc70bcdcf4b..d72279316b88 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1102,3 +1102,223 @@ Value *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
}
return BI->getCondition();
}
+
+// After creating a control flow hub, the operands of PHINodes in an outgoing
+// block Out no longer match the predecessors of that block. Predecessors of Out
+// that are incoming blocks to the hub are now replaced by just one edge from
+// the hub. To match this new control flow, the corresponding values from each
+// PHINode must now be moved a new PHINode in the first guard block of the hub.
+//
+// This operation cannot be performed with SSAUpdater, because it involves one
+// new use: If the block Out is in the list of Incoming blocks, then the newly
+// created PHI in the Hub will use itself along that edge from Out to Hub.
+static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
+ const SetVector<BasicBlock *> &Incoming,
+ BasicBlock *FirstGuardBlock) {
+ auto I = Out->begin();
+ while (I != Out->end() && isa<PHINode>(I)) {
+ auto Phi = cast<PHINode>(I);
+ auto NewPhi =
+ PHINode::Create(Phi->getType(), Incoming.size(),
+ Phi->getName() + ".moved", &FirstGuardBlock->back());
+ for (auto In : Incoming) {
+ Value *V = UndefValue::get(Phi->getType());
+ if (In == Out) {
+ V = NewPhi;
+ } else if (Phi->getBasicBlockIndex(In) != -1) {
+ V = Phi->removeIncomingValue(In, false);
+ }
+ NewPhi->addIncoming(V, In);
+ }
+ assert(NewPhi->getNumIncomingValues() == Incoming.size());
+ if (Phi->getNumOperands() == 0) {
+ Phi->replaceAllUsesWith(NewPhi);
+ I = Phi->eraseFromParent();
+ continue;
+ }
+ Phi->addIncoming(NewPhi, GuardBlock);
+ ++I;
+ }
+}
+
+using BBPredicates = DenseMap<BasicBlock *, PHINode *>;
+using BBSetVector = SetVector<BasicBlock *>;
+
+// Collect predicates for each outgoing block. If control reaches the
+// Hub from an incoming block InBB, then the predicate for each
+// outgoing block OutBB decides whether control is forwarded to OutBB.
+//
+// These predicates are not orthogonal. The Hub evaluates them in the
+// same order as the Outgoing set-vector, and control branches to the
+// first outgoing block whose predicate evaluates to true.
+static void createGuardPredicates(BasicBlock *FirstGuardBlock,
+ BBPredicates &GuardPredicates,
+ SmallVectorImpl<WeakVH> &DeletionCandidates,
+ const BBSetVector &Incoming,
+ const BBSetVector &Outgoing) {
+ auto &Context = Incoming.front()->getContext();
+ auto BoolTrue = ConstantInt::getTrue(Context);
+ auto BoolFalse = ConstantInt::getFalse(Context);
+
+ // The predicate for the last outgoing is trivially true, and so we
+ // process only the first N-1 successors.
+ for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
+ auto Out = Outgoing[i];
+ LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
+ auto Phi =
+ PHINode::Create(Type::getInt1Ty(Context), Incoming.size(),
+ StringRef("Guard.") + Out->getName(), FirstGuardBlock);
+ GuardPredicates[Out] = Phi;
+ }
+
+ for (auto In : Incoming) {
+ auto Branch = cast<BranchInst>(In->getTerminator());
+ BasicBlock *Succ0 = Branch->getSuccessor(0);
+ BasicBlock *Succ1 = nullptr;
+
+ Succ0 = Outgoing.count(Succ0) ? Succ0 : nullptr;
+
+ if (Branch->isUnconditional()) {
+ Branch->setSuccessor(0, FirstGuardBlock);
+ assert(Succ0);
+ } else {
+ Succ1 = Branch->getSuccessor(1);
+ Succ1 = Outgoing.count(Succ1) ? Succ1 : nullptr;
+ assert(Succ0 || Succ1);
+ if (Succ0 && !Succ1) {
+ Branch->setSuccessor(0, FirstGuardBlock);
+ } else if (Succ1 && !Succ0) {
+ Branch->setSuccessor(1, FirstGuardBlock);
+ } else {
+ Branch->eraseFromParent();
+ BranchInst::Create(FirstGuardBlock, In);
+ }
+ }
+
+ assert(Succ0 || Succ1);
+
+ // Optimization: Consider an incoming block A with both successors
+ // Succ0 and Succ1 in the set of outgoing blocks. The predicates
+ // for Succ0 and Succ1 complement each other. If Succ0 is visited
+ // first in the loop below, control will branch to Succ0 using the
+ // corresponding predicate. But if that branch is not taken, then
+ // control must reach Succ1, which means that the predicate for
+ // Succ1 is always true.
+ bool OneSuccessorDone = false;
+ auto Condition = Branch->getCondition();
+ for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
+ auto Out = Outgoing[i];
+ auto Phi = GuardPredicates[Out];
+ if (Out != Succ0 && Out != Succ1) {
+ Phi->addIncoming(BoolFalse, In);
+ continue;
+ }
+ // Optimization: When only one successor is an outgoing block,
+ // the predicate is always true.
+ if (!Succ0 || !Succ1 || OneSuccessorDone) {
+ Phi->addIncoming(BoolTrue, In);
+ continue;
+ }
+ assert(Succ0 && Succ1);
+ OneSuccessorDone = true;
+ if (Out == Succ0) {
+ Phi->addIncoming(Condition, In);
+ continue;
+ }
+ auto Inverted = invertCondition(Condition);
+ DeletionCandidates.push_back(Condition);
+ Phi->addIncoming(Inverted, In);
+ }
+ }
+}
+
+// For each outgoing block OutBB, create a guard block in the Hub. The
+// first guard block was already created outside, and available as the
+// first element in the vector of guard blocks.
+//
+// Each guard block terminates in a conditional branch that transfers
+// control to the corresponding outgoing block or the next guard
+// block. The last guard block has two outgoing blocks as successors
+// since the condition for the final outgoing block is trivially
+// true. So we create one less block (including the first guard block)
+// than the number of outgoing blocks.
+static void createGuardBlocks(SmallVectorImpl<BasicBlock *> &GuardBlocks,
+ Function *F, const BBSetVector &Outgoing,
+ BBPredicates &GuardPredicates, StringRef Prefix) {
+ for (int i = 0, e = Outgoing.size() - 2; i != e; ++i) {
+ GuardBlocks.push_back(
+ BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
+ }
+ assert(GuardBlocks.size() == GuardPredicates.size());
+
+ // To help keep the loop simple, temporarily append the last
+ // outgoing block to the list of guard blocks.
+ GuardBlocks.push_back(Outgoing.back());
+
+ for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
+ auto Out = Outgoing[i];
+ assert(GuardPredicates.count(Out));
+ BranchInst::Create(Out, GuardBlocks[i + 1], GuardPredicates[Out],
+ GuardBlocks[i]);
+ }
+
+ // Remove the last block from the guard list.
+ GuardBlocks.pop_back();
+}
+
+BasicBlock *llvm::CreateControlFlowHub(
+ DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
+ const BBSetVector &Incoming, const BBSetVector &Outgoing,
+ const StringRef Prefix) {
+ auto F = Incoming.front()->getParent();
+ auto FirstGuardBlock =
+ BasicBlock::Create(F->getContext(), Prefix + ".guard", F);
+
+ SmallVector<DominatorTree::UpdateType, 16> Updates;
+ if (DTU) {
+ for (auto In : Incoming) {
+ for (auto Succ : successors(In)) {
+ if (Outgoing.count(Succ))
+ Updates.push_back({DominatorTree::Delete, In, Succ});
+ }
+ Updates.push_back({DominatorTree::Insert, In, FirstGuardBlock});
+ }
+ }
+
+ BBPredicates GuardPredicates;
+ SmallVector<WeakVH, 8> DeletionCandidates;
+ createGuardPredicates(FirstGuardBlock, GuardPredicates, DeletionCandidates,
+ Incoming, Outgoing);
+
+ GuardBlocks.push_back(FirstGuardBlock);
+ createGuardBlocks(GuardBlocks, F, Outgoing, GuardPredicates, Prefix);
+
+ // Update the PHINodes in each outgoing block to match the new control flow.
+ for (int i = 0, e = GuardBlocks.size(); i != e; ++i) {
+ reconnectPhis(Outgoing[i], GuardBlocks[i], Incoming, FirstGuardBlock);
+ }
+ reconnectPhis(Outgoing.back(), GuardBlocks.back(), Incoming, FirstGuardBlock);
+
+ if (DTU) {
+ int NumGuards = GuardBlocks.size();
+ assert((int)Outgoing.size() == NumGuards + 1);
+ for (int i = 0; i != NumGuards - 1; ++i) {
+ Updates.push_back({DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
+ Updates.push_back(
+ {DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
+ }
+ Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
+ Outgoing[NumGuards - 1]});
+ Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
+ Outgoing[NumGuards]});
+ DTU->applyUpdates(Updates);
+ }
+
+ for (auto I : DeletionCandidates) {
+ if (I->use_empty())
+ if (auto Inst = dyn_cast_or_null<Instruction>(I))
+ Inst->eraseFromParent();
+ }
+
+ return FirstGuardBlock;
+}
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 56e5071b2e3f..30a157f2f249 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -62,6 +62,7 @@ add_llvm_component_library(LLVMTransformUtils
StripNonLineTableDebugInfo.cpp
SymbolRewriter.cpp
UnifyFunctionExitNodes.cpp
+ UnifyLoopExits.cpp
Utils.cpp
ValueMapper.cpp
VNCoercion.cpp
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index d5d9dff75ef0..8cccadd45c41 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -3031,3 +3031,42 @@ AllocaInst *llvm::findAllocaForValue(Value *V,
AllocaForValue[V] = Res;
return Res;
}
+
+Value *llvm::invertCondition(Value *Condition) {
+ // First: Check if it's a constant
+ if (Constant *C = dyn_cast<Constant>(Condition))
+ return ConstantExpr::getNot(C);
+
+ // Second: If the condition is already inverted, return the original value
+ Value *NotCondition;
+ if (match(Condition, m_Not(m_Value(NotCondition))))
+ return NotCondition;
+
+ if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
+ // Third: Check all the users for an invert
+ BasicBlock *Parent = Inst->getParent();
+ for (User *U : Condition->users())
+ if (Instruction *I = dyn_cast<Instruction>(U))
+ if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
+ return I;
+
+ // Last option: Create a new instruction
+ auto Inverted = BinaryOperator::CreateNot(Inst, "");
+ if (isa<PHINode>(Inst)) {
+ // FIXME: This fails if the inversion is to be used in a
+ // subsequent PHINode in the same basic block.
+ Inverted->insertBefore(&*Parent->getFirstInsertionPt());
+ } else {
+ Inverted->insertAfter(Inst);
+ }
+ return Inverted;
+ }
+
+ if (Argument *Arg = dyn_cast<Argument>(Condition)) {
+ BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
+ return BinaryOperator::CreateNot(Condition, Arg->getName() + ".inv",
+ &*EntryBlock.getFirstInsertionPt());
+ }
+
+ llvm_unreachable("Unhandled condition to invert");
+}
diff --git a/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
new file mode 100644
index 000000000000..15c951fb1843
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
@@ -0,0 +1,220 @@
+//===- UnifyLoopExits.cpp - Redirect exiting edges to one block -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// For each natural loop with multiple exit blocks, this pass creates a new
+// block N such that all exiting blocks now branch to N, and then control flow
+// is redistributed to all the original exit blocks.
+//
+// Limitation: This assumes that all terminators in the CFG are direct branches
+// (the "br" instruction). The presence of any other control flow
+// such as indirectbr, switch or callbr will cause an assert.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+#define DEBUG_TYPE "unify-loop-exits"
+
+using namespace llvm;
+
+namespace {
+struct UnifyLoopExits : public FunctionPass {
+ static char ID;
+ UnifyLoopExits() : FunctionPass(ID) {
+ initializeUnifyLoopExitsPass(*PassRegistry::getPassRegistry());
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequiredID(LowerSwitchID);
+ AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<DominatorTreeWrapperPass>();
+ AU.addPreservedID(LowerSwitchID);
+ AU.addPreserved<LoopInfoWrapperPass>();
+ AU.addPreserved<DominatorTreeWrapperPass>();
+ }
+
+ bool runOnFunction(Function &F);
+};
+} // namespace
+
+char UnifyLoopExits::ID = 0;
+
+FunctionPass *llvm::createUnifyLoopExitsPass() { return new UnifyLoopExits(); }
+
+INITIALIZE_PASS_BEGIN(UnifyLoopExits, "unify-loop-exits",
+ "Fixup each natural loop to have a single exit block",
+ false /* Only looks at CFG */, false /* Analysis Pass */)
+INITIALIZE_PASS_DEPENDENCY(LowerSwitch)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_END(UnifyLoopExits, "unify-loop-exits",
+ "Fixup each natural loop to have a single exit block",
+ false /* Only looks at CFG */, false /* Analysis Pass */)
+
+// The current transform introduces new control flow paths which may break the
+// SSA requirement that every def must dominate all its uses. For example,
+// consider a value D defined inside the loop that is used by some instruction
+// U outside the loop. It follows that D dominates U, since the original
+// program has valid SSA form. After merging the exits, all paths from D to U
+// now flow through the unified exit block. In addition, there may be other
+// paths that do not pass through D, but now reach the unified exit
+// block. Thus, D no longer dominates U.
+//
+// Restore the dominance by creating a phi for each such D at the new unified
+// loop exit. But when doing this, ignore any uses U that are in the new unified
+// loop exit, since those were introduced specially when the block was created.
+//
+// The use of SSAUpdater seems like overkill for this operation. The location
+// for creating the new PHI is well-known, and also the set of incoming blocks
+// to the new PHI.
+static void restoreSSA(const DominatorTree &DT, const Loop *L,
+ const SetVector<BasicBlock *> &Incoming,
+ BasicBlock *LoopExitBlock) {
+ using InstVector = SmallVector<Instruction *, 8>;
+ using IIMap = DenseMap<Instruction *, InstVector>;
+ IIMap ExternalUsers;
+ for (auto BB : L->blocks()) {
+ for (auto &I : *BB) {
+ for (auto &U : I.uses()) {
+ auto UserInst = cast<Instruction>(U.getUser());
+ auto UserBlock = UserInst->getParent();
+ if (UserBlock == LoopExitBlock)
+ continue;
+ if (L->contains(UserBlock))
+ continue;
+ LLVM_DEBUG(dbgs() << "added ext use for " << I.getName() << "("
+ << BB->getName() << ")"
+ << ": " << UserInst->getName() << "("
+ << UserBlock->getName() << ")"
+ << "\n");
+ ExternalUsers[&I].push_back(UserInst);
+ }
+ }
+ }
+
+ for (auto II : ExternalUsers) {
+ // For each Def used outside the loop, create NewPhi in
+ // LoopExitBlock. NewPhi receives Def only along exiting blocks that
+ // dominate it, while the remaining values are undefined since those paths
+ // didn't exist in the original CFG.
+ auto Def = II.first;
+ LLVM_DEBUG(dbgs() << "externally used: " << Def->getName() << "\n");
+ auto NewPhi = PHINode::Create(Def->getType(), Incoming.size(),
+ Def->getName() + ".moved",
+ LoopExitBlock->getTerminator());
+ for (auto In : Incoming) {
+ LLVM_DEBUG(dbgs() << "predecessor " << In->getName() << ": ");
+ if (Def->getParent() == In || DT.dominates(Def, In)) {
+ LLVM_DEBUG(dbgs() << "dominated\n");
+ NewPhi->addIncoming(Def, In);
+ } else {
+ LLVM_DEBUG(dbgs() << "not dominated\n");
+ NewPhi->addIncoming(UndefValue::get(Def->getType()), In);
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "external users:");
+ for (auto U : II.second) {
+ LLVM_DEBUG(dbgs() << " " << U->getName());
+ U->replaceUsesOfWith(Def, NewPhi);
+ }
+ LLVM_DEBUG(dbgs() << "\n");
+ }
+}
+
+static bool unifyLoopExits(DominatorTree &DT, LoopInfo &LI, Loop *L) {
+ // To unify the loop exits, we need a list of the exiting blocks as
+ // well as exit blocks. The functions for locating these lists both
+ // traverse the entire loop body. It is more efficient to first
+ // locate the exiting blocks and then examine their successors to
+ // locate the exit blocks.
+ SetVector<BasicBlock *> ExitingBlocks;
+ SetVector<BasicBlock *> Exits;
+
+ // We need SetVectors, but the Loop API takes a vector, so we use a temporary.
+ SmallVector<BasicBlock *, 8> Temp;
+ L->getExitingBlocks(Temp);
+ for (auto BB : Temp) {
+ ExitingBlocks.insert(BB);
+ for (auto S : successors(BB)) {
+ auto SL = LI.getLoopFor(S);
+ // A successor is not an exit if it is directly or indirectly in the
+ // current loop.
+ if (SL == L || L->contains(SL))
+ continue;
+ Exits.insert(S);
+ }
+ }
+
+ LLVM_DEBUG(
+ dbgs() << "Found exit blocks:";
+ for (auto Exit : Exits) {
+ dbgs() << " " << Exit->getName();
+ }
+ dbgs() << "\n";
+
+ dbgs() << "Found exiting blocks:";
+ for (auto EB : ExitingBlocks) {
+ dbgs() << " " << EB->getName();
+ }
+ dbgs() << "\n";);
+
+ if (Exits.size() <= 1) {
+ LLVM_DEBUG(dbgs() << "loop does not have multiple exits; nothing to do\n");
+ return false;
+ }
+
+ SmallVector<BasicBlock *, 8> GuardBlocks;
+ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
+ auto LoopExitBlock = CreateControlFlowHub(&DTU, GuardBlocks, ExitingBlocks,
+ Exits, "loop.exit");
+
+ restoreSSA(DT, L, ExitingBlocks, LoopExitBlock);
+
+#if defined(EXPENSIVE_CHECKS)
+ assert(DT.verify(DominatorTree::VerificationLevel::Full));
+#else
+ assert(DT.verify(DominatorTree::VerificationLevel::Full));
+#endif // EXPENSIVE_CHECKS
+ L->verifyLoop();
+
+ // The guard blocks were created outside the loop, so they need to become
+ // members of the parent loop.
+ if (auto ParentLoop = L->getParentLoop()) {
+ for (auto G : GuardBlocks) {
+ ParentLoop->addBasicBlockToLoop(G, LI);
+ }
+ ParentLoop->verifyLoop();
+ }
+
+#if defined(EXPENSIVE_CHECKS)
+ LI.verify(DT);
+#endif // EXPENSIVE_CHECKS
+
+ return true;
+}
+
+bool UnifyLoopExits::runOnFunction(Function &F) {
+ LLVM_DEBUG(dbgs() << "===== Unifying loop exits in function " << F.getName()
+ << "\n");
+ auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+
+ bool Changed = false;
+ auto Loops = LI.getLoopsInPreorder();
+ for (auto L : Loops) {
+ LLVM_DEBUG(dbgs() << "Loop: " << L->getHeader()->getName() << " (depth: "
+ << LI.getLoopDepth(L->getHeader()) << ")\n");
+ Changed |= unifyLoopExits(DT, LI, L);
+ }
+ return Changed;
+}
diff --git a/llvm/lib/Transforms/Utils/Utils.cpp b/llvm/lib/Transforms/Utils/Utils.cpp
index 7769c7493cda..363110ab8a7d 100644
--- a/llvm/lib/Transforms/Utils/Utils.cpp
+++ b/llvm/lib/Transforms/Utils/Utils.cpp
@@ -40,6 +40,7 @@ void llvm::initializeTransformUtils(PassRegistry &Registry) {
initializeStripGCRelocatesPass(Registry);
initializePredicateInfoPrinterLegacyPassPass(Registry);
initializeInjectTLIMappingsLegacyPass(Registry);
+ initializeUnifyLoopExitsPass(Registry);
}
/// LLVMInitializeTransformUtils - C binding for initializeTransformUtilsPasses.
diff --git a/llvm/test/CodeGen/AMDGPU/loop_break.ll b/llvm/test/CodeGen/AMDGPU/loop_break.ll
index 6e2868a94167..4b3fb1aa0791 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_break.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_break.ll
@@ -499,8 +499,8 @@ define amdgpu_kernel void @invert_true_phi_cond_break_loop(i32 %arg) #0 {
; GCN-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GCN-NEXT: BB5_3: ; %Flow
; GCN-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-NEXT: s_add_i32 s6, s6, 1
; GCN-NEXT: s_xor_b64 s[8:9], s[4:5], -1
+; GCN-NEXT: s_add_i32 s6, s6, 1
; GCN-NEXT: s_and_b64 s[8:9], exec, s[8:9]
; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[0:1]
; GCN-NEXT: s_andn2_b64 exec, exec, s[0:1]
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
new file mode 100644
index 000000000000..32565503f73e
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -structurizecfg -S | FileCheck %s
+
+; The structurizer uses an RPO traversal over a region, along with a
+; manual hack that is meant to sort ensure that blocks within a loop
+; are all visited before visiting blocks outside the loop. But this
+; does not always work as expected. For example the results are
+; incorrect when multiple nested loops are involved.
+
+; The workaround for this is to unify loop exits. Each loop now
+; becomes an SESE region with a single header and a single exit. The
+; structurizer is a region pass, and it no longer sees the entire loop
+; nest in a single region. More importantly, for each loop, the only
+; block reachable outside the loop is the region exit, which avoids
+; any confusion in the hacked RPO traversal.
+
+; In the function below, B1 is an exiting block in outer loop H1. It's
+; successor inside the loop is the header of another loop H2. Due to
+; the incorrect traversal, B1 dominates all the blocks in the
+; structurized program, except the header H1.
+
+define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
+; CHECK-LABEL: @exiting-block(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[PREDH1_INV:%.*]] = xor i1 [[PREDH1:%.*]], true
+; CHECK-NEXT: [[PREDB2_INV:%.*]] = xor i1 [[PREDB2:%.*]], true
+; CHECK-NEXT: br label [[H1:%.*]]
+; CHECK: H1:
+; CHECK-NEXT: br i1 [[PREDH1_INV]], label [[B1:%.*]], label [[FLOW3:%.*]]
+; CHECK: Flow3:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[PREDB1:%.*]], [[B1]] ], [ [[PREDH1]], [[H1]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label [[H2:%.*]], label [[FLOW4:%.*]]
+; CHECK: H2:
+; CHECK-NEXT: br i1 [[PREDH2:%.*]], label [[B2:%.*]], label [[FLOW:%.*]]
+; CHECK: B2:
+; CHECK-NEXT: br i1 [[PREDB2_INV]], label [[L2:%.*]], label [[FLOW2:%.*]]
+; CHECK: Flow:
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[TMP4:%.*]], [[FLOW2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label [[LOOP_EXIT_GUARD1:%.*]], label [[H2]]
+; CHECK: L2:
+; CHECK-NEXT: br label [[FLOW2]]
+; CHECK: L1:
+; CHECK-NEXT: br label [[FLOW5:%.*]]
+; CHECK: B1:
+; CHECK-NEXT: br label [[FLOW3]]
+; CHECK: C:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+; CHECK: Flow5:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ false, [[L1:%.*]] ], [ true, [[LOOP_EXIT_GUARD1]] ]
+; CHECK-NEXT: br label [[FLOW4]]
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: br i1 [[TMP5:%.*]], label [[C:%.*]], label [[EXIT]]
+; CHECK: Flow2:
+; CHECK-NEXT: [[TMP4]] = phi i1 [ false, [[L2]] ], [ true, [[B2]] ]
+; CHECK-NEXT: br label [[FLOW]]
+; CHECK: Flow4:
+; CHECK-NEXT: [[TMP5]] = phi i1 [ false, [[FLOW5]] ], [ true, [[FLOW3]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ [[TMP3]], [[FLOW5]] ], [ true, [[FLOW3]] ]
+; CHECK-NEXT: br i1 [[TMP6]], label [[LOOP_EXIT_GUARD:%.*]], label [[H1]]
+; CHECK: loop.exit.guard1:
+; CHECK-NEXT: br i1 [[TMP1]], label [[L1]], label [[FLOW5]]
+;
+entry:
+ br label %H1
+
+H1: ; preds = %L1, %entry
+ br i1 %PredH1, label %H2, label %B1
+
+H2: ; preds = %B1, %L2, %H1
+ br i1 %PredH2, label %B2, label %L1
+
+B2: ; preds = %H2
+ br i1 %PredB2, label %exit, label %L2
+
+L2: ; preds = %B2
+ br label %H2
+
+L1: ; preds = %H2
+ br label %H1
+
+B1: ; preds = %H1
+ br i1 %PredB1, label %H2, label %C
+
+C: ; preds = %B1
+ br label %exit
+
+exit: ; preds = %C, %B2
+ ret void
+}
+
+; The function below has three nested loops. Due to the incorrect
+; traversal, H2 dominates H3 in the structurized program, and the
+; backedge from L13 to H3 has no equivalent path.
+
+define void @incorrect-backedge(i1 %PredH2, i1 %PredH3, i1 %PredL2, i1 %PredL13, i1 %PredL1)
+; CHECK-LABEL: @incorrect-backedge(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[PREDH2_INV:%.*]] = xor i1 [[PREDH2:%.*]], true
+; CHECK-NEXT: [[PREDL2_INV:%.*]] = xor i1 [[PREDL2:%.*]], true
+; CHECK-NEXT: [[PREDH3_INV:%.*]] = xor i1 [[PREDH3:%.*]], true
+; CHECK-NEXT: [[PREDL13_INV:%.*]] = xor i1 [[PREDL13:%.*]], true
+; CHECK-NEXT: br label [[H1:%.*]]
+; CHECK: H1:
+; CHECK-NEXT: br label [[H2:%.*]]
+; CHECK: H2:
+; CHECK-NEXT: br i1 [[PREDH2_INV]], label [[H3:%.*]], label [[FLOW4:%.*]]
+; CHECK: H3:
+; CHECK-NEXT: br i1 [[PREDH3_INV]], label [[L2:%.*]], label [[FLOW:%.*]]
+; CHECK: L2:
+; CHECK-NEXT: br i1 [[PREDL2_INV]], label [[L13:%.*]], label [[FLOW3:%.*]]
+; CHECK: Flow:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ [[TMP6:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label [[LOOP_EXIT_GUARD2:%.*]], label [[H3]]
+; CHECK: L13:
+; CHECK-NEXT: br label [[FLOW3]]
+; CHECK: Flow5:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ [[TMP8:%.*]], [[LOOP_EXIT_GUARD1:%.*]] ], [ true, [[LOOP_EXIT_GUARD:%.*]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, [[LOOP_EXIT_GUARD1]] ], [ true, [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT: br i1 [[TMP4]], label [[L1:%.*]], label [[FLOW6:%.*]]
+; CHECK: L1:
+; CHECK-NEXT: br label [[FLOW6]]
+; CHECK: Flow6:
+; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ [[PREDL1:%.*]], [[L1]] ], [ [[TMP3]], [[FLOW5:%.*]] ]
+; CHECK-NEXT: br i1 [[TMP5]], label [[EXIT:%.*]], label [[H1]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: br i1 [[TMP11:%.*]], label [[LOOP_EXIT_GUARD1]], label [[FLOW5]]
+; CHECK: loop.exit.guard1:
+; CHECK-NEXT: br label [[FLOW5]]
+; CHECK: Flow3:
+; CHECK-NEXT: [[TMP6]] = phi i1 [ true, [[L13]] ], [ false, [[L2]] ]
+; CHECK-NEXT: [[TMP7]] = phi i1 [ [[PREDL13_INV]], [[L13]] ], [ true, [[L2]] ]
+; CHECK-NEXT: br label [[FLOW]]
+; CHECK: Flow4:
+; CHECK-NEXT: [[TMP8]] = phi i1 [ [[TMP0]], [[LOOP_EXIT_GUARD2]] ], [ false, [[H2]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ false, [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ [[TMP1]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: [[TMP11]] = xor i1 [[TMP9]], true
+; CHECK-NEXT: br i1 [[TMP10]], label [[LOOP_EXIT_GUARD]], label [[H2]]
+; CHECK: loop.exit.guard2:
+; CHECK-NEXT: br label [[FLOW4]]
+;
+{
+entry:
+ br label %H1
+
+H1:
+ br label %H2
+
+H2:
+ br i1 %PredH2, label %L1, label %H3
+
+H3:
+ br i1 %PredH3, label %exit, label %L2
+
+L2:
+ br i1 %PredL2, label %H2, label %L13
+
+L13:
+ br i1 %PredL13, label %H3, label %H1
+
+L1:
+ br i1 %PredL1, label %exit, label %H1
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/basic.ll b/llvm/test/Transforms/UnifyLoopExits/basic.ll
new file mode 100644
index 000000000000..11fa32c86243
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/basic.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+define void @loop_1(i1 %PredEntry, i1 %PredB, i1 %PredC, i1 %PredD) {
+; CHECK-LABEL: @loop_1(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[G:%.*]]
+; CHECK: A:
+; CHECK-NEXT: br label [[B:%.*]]
+; CHECK: B:
+; CHECK-NEXT: br i1 [[PREDB:%.*]], label [[C:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: C:
+; CHECK-NEXT: br i1 [[PREDC:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK: D:
+; CHECK-NEXT: br i1 [[PREDD:%.*]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: E:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: F:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: G:
+; CHECK-NEXT: br label [[F:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_E:%.*]] = phi i1 [ true, [[B]] ], [ false, [[C]] ], [ false, [[D]] ]
+; CHECK-NEXT: br i1 [[GUARD_E]], label [[E:%.*]], label [[F]]
+;
+entry:
+ br i1 %PredEntry, label %A, label %G
+
+A:
+ br label %B
+
+B:
+ br i1 %PredB, label %C, label %E
+
+C:
+ br i1 %PredC, label %D, label %F
+
+D:
+ br i1 %PredD, label %A, label %F
+
+E:
+ br label %exit
+
+F:
+ br label %exit
+
+G:
+ br label %F
+
+exit:
+ ret void
+}
+
+define void @loop_2(i1 %PredA, i1 %PredB, i1 %PredC) {
+; CHECK-LABEL: @loop_2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[A:%.*]]
+; CHECK: A:
+; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: B:
+; CHECK-NEXT: br i1 [[PREDB:%.*]], label [[C:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK: C:
+; CHECK-NEXT: br i1 [[PREDC:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK: D:
+; CHECK-NEXT: br label [[A]]
+; CHECK: X:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: Y:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: Z:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_X:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ], [ false, [[C]] ]
+; CHECK-NEXT: [[GUARD_Y:%.*]] = phi i1 [ false, [[A]] ], [ true, [[B]] ], [ false, [[C]] ]
+; CHECK-NEXT: br i1 [[GUARD_X]], label [[X:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; CHECK: loop.exit.guard1:
+; CHECK-NEXT: br i1 [[GUARD_Y]], label [[Y:%.*]], label [[Z:%.*]]
+;
+entry:
+ br label %A
+
+A:
+ br i1 %PredA, label %B, label %X
+
+B:
+ br i1 %PredB, label %C, label %Y
+
+C:
+ br i1 %PredC, label %D, label %Z
+
+D:
+ br label %A
+
+X:
+ br label %exit
+
+Y:
+ br label %exit
+
+Z:
+ br label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/nested.ll b/llvm/test/Transforms/UnifyLoopExits/nested.ll
new file mode 100644
index 000000000000..e5d04dccbbdf
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/nested.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+define void @nested(i1 %PredB3, i1 %PredB4, i1 %PredA4, i1 %PredA3, i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @nested(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[A1:%.*]]
+; CHECK: A1:
+; CHECK-NEXT: br label [[B1:%.*]]
+; CHECK: B1:
+; CHECK-NEXT: br label [[B2:%.*]]
+; CHECK: B2:
+; CHECK-NEXT: [[X_INC:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT: br label [[B3:%.*]]
+; CHECK: B3:
+; CHECK-NEXT: br i1 [[PREDB3:%.*]], label [[B4:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; CHECK: B4:
+; CHECK-NEXT: br i1 [[PREDB4:%.*]], label [[B1]], label [[LOOP_EXIT_GUARD1]]
+; CHECK: A2:
+; CHECK-NEXT: br label [[A4:%.*]]
+; CHECK: A3:
+; CHECK-NEXT: br label [[A4]]
+; CHECK: A4:
+; CHECK-NEXT: [[A4_PHI:%.*]] = phi i32 [ [[Y:%.*]], [[A3:%.*]] ], [ [[X_INC_MOVED:%.*]], [[A2:%.*]] ]
+; CHECK-NEXT: br i1 [[PREDA4:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[A5:%.*]]
+; CHECK: A5:
+; CHECK-NEXT: br i1 [[PREDA3:%.*]], label [[LOOP_EXIT_GUARD]], label [[A1]]
+; CHECK: C:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[EXIT_PHI:%.*]] = phi i32 [ [[Z:%.*]], [[C:%.*]] ], [ [[EXIT_PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[A4]] ], [ false, [[A5]] ]
+; CHECK-NEXT: [[EXIT_PHI_MOVED]] = phi i32 [ undef, [[A4]] ], [ [[A4_PHI]], [[A5]] ]
+; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[EXIT]]
+; CHECK: loop.exit.guard1:
+; CHECK-NEXT: [[GUARD_A3:%.*]] = phi i1 [ true, [[B3]] ], [ false, [[B4]] ]
+; CHECK-NEXT: [[X_INC_MOVED]] = phi i32 [ [[X_INC]], [[B3]] ], [ [[X_INC]], [[B4]] ]
+; CHECK-NEXT: br i1 [[GUARD_A3]], label [[A3]], label [[A2]]
+;
+entry:
+ br label %A1
+
+A1:
+ br label %B1
+
+B1:
+ br label %B2
+
+B2:
+ %X.inc = add i32 %X, 1
+ br label %B3
+
+B3:
+ br i1 %PredB3, label %B4, label %A3
+
+B4:
+ br i1 %PredB4, label %B1, label %A2
+
+A2:
+ br label %A4
+
+A3:
+ br label %A4
+
+A4:
+ %A4.phi = phi i32 [%Y, %A3], [%X.inc, %A2]
+ br i1 %PredA4, label %C, label %A5
+
+A5:
+ br i1 %PredA3, label %exit, label %A1
+
+C:
+ br label %exit
+
+exit:
+ %exit.phi = phi i32 [%A4.phi, %A5], [%Z, %C]
+ ret void
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
new file mode 100644
index 000000000000..2db170cd4889
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+; Loop consists of A and B:
+; - A is the header
+; - A and B are exiting blocks
+; - C and return are exit blocks.
+; Pattern: Value (%mytmp42) defined in exiting block (A) and used in
+; exit block (return).
+; The relevant code uses DT::dominates(Value,
+; BasicBlock). This is misnamed because it actually checks
+; strict dominance, causing the pattern to be miscompiled
+; (the use receives an undef value).
+define i32 @exiting-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @exiting-used-in-exit(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[A:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: B:
+; CHECK-NEXT: [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MYTMP41]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: C:
+; CHECK-NEXT: [[INC:%.*]] = add i32 [[MYTMP41_MOVED:%.*]], 1
+; CHECK-NEXT: br label [[RETURN:%.*]]
+; CHECK: return:
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[INC]], [[C:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT: ret i32 [[PHI]]
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ]
+; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP42]], [[A]] ], [ undef, [[B]] ]
+; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[B]] ]
+; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[C]]
+;
+entry:
+ br label %A
+
+A:
+ %mytmp42 = load i32, i32* %arg1, align 4
+ %cmp1 = icmp slt i32 %mytmp42, 0
+ br i1 %cmp1, label %B, label %return
+
+B:
+ %mytmp41 = load i32, i32* %arg2, align 4
+ %cmp = icmp slt i32 %mytmp41, 0
+ br i1 %cmp, label %A, label %C
+
+C:
+ %inc = add i32 %mytmp41, 1
+ br label %return
+
+return:
+ %phi = phi i32 [ %inc, %C ], [ %mytmp42, %A ]
+ ret i32 %phi
+}
+
+; Loop consists of A, B and C:
+; - A is the header
+; - A and C are exiting blocks
+; - B is an "internal" block that dominates exiting block C
+; - D and return are exit blocks.
+; Pattern: Value (%mytmp41) defined in internal block (B) and used in an
+; exit block (D).
+define i32 @internal-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @internal-used-in-exit(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT: br label [[A:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: B:
+; CHECK-NEXT: [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT: br label [[C:%.*]]
+; CHECK: C:
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: D:
+; CHECK-NEXT: [[INC:%.*]] = add i32 [[MYTMP41_MOVED:%.*]], 1
+; CHECK-NEXT: br label [[RETURN:%.*]]
+; CHECK: return:
+; CHECK-NEXT: ret i32 0
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D:%.*]]
+;
+entry:
+ %mytmp42 = load i32, i32* %arg1, align 4
+ br label %A
+
+A:
+ %cmp1 = icmp slt i32 %mytmp42, 0
+ br i1 %cmp1, label %B, label %return
+
+B:
+ %mytmp41 = load i32, i32* %arg2, align 4
+ br label %C
+
+C:
+ %cmp = icmp slt i32 %mytmp42, 0
+ br i1 %cmp, label %A, label %D
+
+D:
+ %inc = add i32 %mytmp41, 1
+ br label %return
+
+return:
+ ret i32 0
+}
+
+; Loop consists of A, B and C:
+; - A is the header
+; - A and C are exiting blocks
+; - B is an "internal" block that dominates exiting block C
+; - D and return are exit blocks.
+; Pattern: %return contains a phi node that receives values from
+; %entry, %A and %D. This mixes all the special cases in a single phi.
+define i32 @mixed-use-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @mixed-use-in-exit(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP2]], label [[A:%.*]], label [[RETURN:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[MYTMP43:%.*]] = add i32 [[MYTMP42]], 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: B:
+; CHECK-NEXT: [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT: br label [[C:%.*]]
+; CHECK: C:
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: D:
+; CHECK-NEXT: br label [[RETURN]]
+; CHECK: return:
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[ENTRY:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT: ret i32 [[PHI]]
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP43]], [[A]] ], [ undef, [[C]] ]
+; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D]]
+;
+entry:
+ %mytmp42 = load i32, i32* %arg1, align 4
+ %cmp2 = icmp slt i32 %mytmp42, 0
+ br i1 %cmp2, label %A, label %return
+
+A:
+ %mytmp43 = add i32 %mytmp42, 1
+ %cmp1 = icmp slt i32 %mytmp42, 0
+ br i1 %cmp1, label %B, label %return
+
+B:
+ %mytmp41 = load i32, i32* %arg2, align 4
+ br label %C
+
+C:
+ %cmp = icmp slt i32 %mytmp42, 0
+ br i1 %cmp, label %A, label %D
+
+D:
+ br label %return
+
+return:
+ %phi = phi i32 [ %mytmp41, %D ], [ %mytmp43, %A ], [%mytmp42, %entry]
+ ret i32 %phi
+}
+
+; Loop consists of A, B and C:
+; - A is the header
+; - A and C are exiting blocks
+; - B is an "internal" block that dominates exiting block C
+; - D and E are exit blocks.
+; Pattern: Value (%mytmp41) defined in internal block (B) and used in a
+; downstream block not related to the loop (return). The use
+; is a phi where the incoming block for %mytmp41 is not related
+; to the loop (D).
+; This pattern does not involve either the exiting blocks or
+; the exit blocks, which catches any such assumptions built
+; into the SSA reconstruction phase.
+define i32 @phi-via-external-block(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @phi-via-external-block(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT: br label [[A:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: B:
+; CHECK-NEXT: [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT: br label [[C:%.*]]
+; CHECK: C:
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: D:
+; CHECK-NEXT: br label [[RETURN:%.*]]
+; CHECK: E:
+; CHECK-NEXT: br label [[RETURN]]
+; CHECK: return:
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[E:%.*]] ]
+; CHECK-NEXT: ret i32 [[PHI]]
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_E:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT: br i1 [[GUARD_E]], label [[E]], label [[D]]
+;
+entry:
+ %mytmp42 = load i32, i32* %arg1, align 4
+ br label %A
+
+A:
+ %cmp1 = icmp slt i32 %mytmp42, 0
+ br i1 %cmp1, label %B, label %E
+
+B:
+ %mytmp41 = load i32, i32* %arg2, align 4
+ br label %C
+
+C:
+ %cmp = icmp slt i32 %mytmp42, 0
+ br i1 %cmp, label %A, label %D
+
+D:
+ br label %return
+
+E:
+ br label %return
+
+return:
+ %phi = phi i32 [ %mytmp41, %D ], [ %mytmp42, %E ]
+ ret i32 %phi
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/switch.ll b/llvm/test/Transforms/UnifyLoopExits/switch.ll
new file mode 100644
index 000000000000..618e54d95a7e
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/switch.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+define void @loop_1(i32 %Value, i1 %PredEntry, i1 %PredD) {
+; CHECK-LABEL: @loop_1(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[G:%.*]]
+; CHECK: A:
+; CHECK-NEXT: br label [[B:%.*]]
+; CHECK: B:
+; CHECK-NEXT: br label [[NODEBLOCK:%.*]]
+; CHECK: NodeBlock:
+; CHECK-NEXT: [[PIVOT:%.*]] = icmp slt i32 [[VALUE:%.*]], 1
+; CHECK-NEXT: br i1 [[PIVOT]], label [[LEAFBLOCK:%.*]], label [[LEAFBLOCK1:%.*]]
+; CHECK: LeafBlock1:
+; CHECK-NEXT: [[SWITCHLEAF2:%.*]] = icmp eq i32 [[VALUE]], 1
+; CHECK-NEXT: br i1 [[SWITCHLEAF2]], label [[D:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: LeafBlock:
+; CHECK-NEXT: [[SWITCHLEAF:%.*]] = icmp eq i32 [[VALUE]], 0
+; CHECK-NEXT: br i1 [[SWITCHLEAF]], label [[C:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK: C:
+; CHECK-NEXT: br label [[D]]
+; CHECK: D:
+; CHECK-NEXT: br i1 [[PREDD:%.*]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: NewDefault:
+; CHECK-NEXT: br label [[X:%.*]]
+; CHECK: X:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: Y:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: G:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[GUARD_NEWDEFAULT:%.*]] = phi i1 [ true, [[LEAFBLOCK1]] ], [ true, [[LEAFBLOCK]] ], [ false, [[D]] ]
+; CHECK-NEXT: br i1 [[GUARD_NEWDEFAULT]], label [[NEWDEFAULT:%.*]], label [[Y:%.*]]
+;
+entry:
+ br i1 %PredEntry, label %A, label %G
+
+A:
+ br label %B
+
+B:
+ switch i32 %Value, label %X [
+ i32 0, label %C
+ i32 1, label %D
+ ]
+
+C:
+ br label %D
+
+D:
+ br i1 %PredD, label %A, label %Y
+
+X:
+ br label %exit
+
+Y:
+ br label %exit
+
+G:
+ br label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 76dd071bd2c7..cd02971b2494 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -319,6 +319,7 @@ int main(int argc, char **argv) {
initializeScalarizeMaskedMemIntrinPass(*Registry);
initializeExpandReductionsPass(*Registry);
initializeHardwareLoopsPass(*Registry);
+ initializeTransformUtils(*Registry);
// Initialize debugging passes.
initializeScavengerTestPass(*Registry);
More information about the llvm-commits
mailing list