[llvm] f59205a - [BasicBlockUtils] Add a new way for CreateControlFlowHub()
Brendon Cahoon via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 31 07:03:51 PDT 2022
Author: Brendon Cahoon
Date: 2022-10-31T08:58:54-05:00
New Revision: f59205aef957da9017285d4c0bb6f9c6f244621f
URL: https://github.com/llvm/llvm-project/commit/f59205aef957da9017285d4c0bb6f9c6f244621f
DIFF: https://github.com/llvm/llvm-project/commit/f59205aef957da9017285d4c0bb6f9c6f244621f.diff
LOG: [BasicBlockUtils] Add a new way for CreateControlFlowHub()
The existing way of creating the predicate in the guard blocks uses
a boolean value per outgoing block. This increases the number of live
booleans as the number of outgoing blocks increases. The new way added
in this change is to store one integer to represent the outgoing block
we want to branch to, then at each guard block, an integer equality
check is performed to decide which a specific outgoing block is taken.
Using an integer reduces the number of live values and decreases
register pressure especially in cases where there are a large number
of outgoing blocks. The integer based approach is used when the
number of outgoing blocks crosses a threshold, which is currently set
to 32.
Patch by Ruiling Song.
Differential review: https://reviews.llvm.org/D127831
Added:
llvm/test/Transforms/UnifyLoopExits/integer_guards.ll
Modified:
llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
llvm/test/Transforms/FixIrreducible/basic.ll
llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
llvm/test/Transforms/UnifyLoopExits/nested.ll
llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index fcdd2aa0e0601..fc6b0cbab056d 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -575,10 +575,11 @@ bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI,
/// may have additional information which simplifies this fixup. For example,
/// see restoreSSA() in the UnifyLoopExits pass.
BasicBlock *CreateControlFlowHub(DomTreeUpdater *DTU,
- SmallVectorImpl<BasicBlock *> &GuardBlocks,
- const SetVector<BasicBlock *> &Predecessors,
- const SetVector<BasicBlock *> &Successors,
- const StringRef Prefix);
+ SmallVectorImpl<BasicBlock *> &GuardBlocks,
+ const SetVector<BasicBlock *> &Predecessors,
+ const SetVector<BasicBlock *> &Successors,
+ const StringRef Prefix,
+ Optional<unsigned> MaxControlFlowBooleans = None);
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index d31c75161e1f4..56b2f5796681e 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1591,7 +1591,7 @@ static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
auto Phi = cast<PHINode>(I);
auto NewPhi =
PHINode::Create(Phi->getType(), Incoming.size(),
- Phi->getName() + ".moved", &FirstGuardBlock->back());
+ Phi->getName() + ".moved", &FirstGuardBlock->front());
for (auto *In : Incoming) {
Value *V = UndefValue::get(Phi->getType());
if (In == Out) {
@@ -1612,7 +1612,7 @@ static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
}
}
-using BBPredicates = DenseMap<BasicBlock *, PHINode *>;
+using BBPredicates = DenseMap<BasicBlock *, Instruction *>;
using BBSetVector = SetVector<BasicBlock *>;
// Redirects the terminator of the incoming block to the first guard
@@ -1683,30 +1683,60 @@ static void setupBranchForGuard(SmallVectorImpl<BasicBlock *> &GuardBlocks,
GuardBlocks.pop_back();
}
-// Capture the existing control flow as guard predicates, and redirect
-// control flow from \p Incoming block through the \p GuardBlocks to the
-// \p Outgoing blocks.
-//
-// There is one guard predicate for each outgoing block OutBB. The
-// predicate represents whether the hub should transfer control flow
-// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates
-// them in the same order as the Outgoing set-vector, and control
-// branches to the first outgoing block whose predicate evaluates to true.
-static void
-convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
- SmallVectorImpl<WeakVH> &DeletionCandidates,
- const BBSetVector &Incoming,
- const BBSetVector &Outgoing, const StringRef Prefix) {
- BBPredicates GuardPredicates;
- auto F = Incoming.front()->getParent();
+/// We are using one integer to represent the block we are branching to. Then at
+/// each guard block, the predicate was calcuated using a simple `icmp eq`.
+static void calcPredicateUsingInteger(
+ const BBSetVector &Incoming, const BBSetVector &Outgoing,
+ SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates) {
auto &Context = Incoming.front()->getContext();
- auto BoolTrue = ConstantInt::getTrue(Context);
- auto BoolFalse = ConstantInt::getFalse(Context);
+ auto FirstGuardBlock = GuardBlocks.front();
- for (int i = 0, e = Outgoing.size() - 1; i != e; ++i)
- GuardBlocks.push_back(
- BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
+ auto Phi = PHINode::Create(Type::getInt32Ty(Context), Incoming.size(),
+ "merged.bb.idx", FirstGuardBlock);
+
+ for (auto In : Incoming) {
+ Value *Condition;
+ BasicBlock *Succ0;
+ BasicBlock *Succ1;
+ std::tie(Condition, Succ0, Succ1) =
+ redirectToHub(In, FirstGuardBlock, Outgoing);
+ Value *IncomingId = nullptr;
+ if (Succ0 && Succ1) {
+ // target_bb_index = Condition ? index_of_succ0 : index_of_succ1.
+ auto Succ0Iter = find(Outgoing, Succ0);
+ auto Succ1Iter = find(Outgoing, Succ1);
+ Value *Id0 = ConstantInt::get(Type::getInt32Ty(Context),
+ std::distance(Outgoing.begin(), Succ0Iter));
+ Value *Id1 = ConstantInt::get(Type::getInt32Ty(Context),
+ std::distance(Outgoing.begin(), Succ1Iter));
+ IncomingId = SelectInst::Create(Condition, Id0, Id1, "target.bb.idx",
+ In->getTerminator());
+ } else {
+ // Get the index of the non-null successor.
+ auto SuccIter = Succ0 ? find(Outgoing, Succ0) : find(Outgoing, Succ1);
+ IncomingId = ConstantInt::get(Type::getInt32Ty(Context),
+ std::distance(Outgoing.begin(), SuccIter));
+ }
+ Phi->addIncoming(IncomingId, In);
+ }
+
+ for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
+ auto Out = Outgoing[i];
+ auto Cmp = ICmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, Phi,
+ ConstantInt::get(Type::getInt32Ty(Context), i),
+ Out->getName() + ".predicate", GuardBlocks[i]);
+ GuardPredicates[Out] = Cmp;
+ }
+}
+/// We record the predicate of each outgoing block using a phi of boolean.
+static void calcPredicateUsingBooleans(
+ const BBSetVector &Incoming, const BBSetVector &Outgoing,
+ SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates,
+ SmallVectorImpl<WeakVH> &DeletionCandidates) {
+ auto &Context = Incoming.front()->getContext();
+ auto BoolTrue = ConstantInt::getTrue(Context);
+ auto BoolFalse = ConstantInt::getFalse(Context);
auto FirstGuardBlock = GuardBlocks.front();
// The predicate for the last outgoing is trivially true, and so we
@@ -1738,7 +1768,7 @@ convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
bool OneSuccessorDone = false;
for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
auto Out = Outgoing[i];
- auto Phi = GuardPredicates[Out];
+ PHINode *Phi = cast<PHINode>(GuardPredicates[Out]);
if (Out != Succ0 && Out != Succ1) {
Phi->addIncoming(BoolFalse, In);
} else if (!Succ0 || !Succ1 || OneSuccessorDone) {
@@ -1758,13 +1788,48 @@ convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
}
}
}
+}
+
+// Capture the existing control flow as guard predicates, and redirect
+// control flow from \p Incoming block through the \p GuardBlocks to the
+// \p Outgoing blocks.
+//
+// There is one guard predicate for each outgoing block OutBB. The
+// predicate represents whether the hub should transfer control flow
+// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates
+// them in the same order as the Outgoing set-vector, and control
+// branches to the first outgoing block whose predicate evaluates to true.
+static void
+convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
+ SmallVectorImpl<WeakVH> &DeletionCandidates,
+ const BBSetVector &Incoming,
+ const BBSetVector &Outgoing, const StringRef Prefix,
+ Optional<unsigned> MaxControlFlowBooleans) {
+ BBPredicates GuardPredicates;
+ auto F = Incoming.front()->getParent();
+
+ for (int i = 0, e = Outgoing.size() - 1; i != e; ++i)
+ GuardBlocks.push_back(
+ BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
+
+ // When we are using an integer to record which target block to jump to, we
+ // are creating less live values, actually we are using one single integer to
+ // store the index of the target block. When we are using booleans to store
+ // the branching information, we need (N-1) boolean values, where N is the
+ // number of outgoing block.
+ if (!MaxControlFlowBooleans || Outgoing.size() <= *MaxControlFlowBooleans)
+ calcPredicateUsingBooleans(Incoming, Outgoing, GuardBlocks, GuardPredicates,
+ DeletionCandidates);
+ else
+ calcPredicateUsingInteger(Incoming, Outgoing, GuardBlocks, GuardPredicates);
+
setupBranchForGuard(GuardBlocks, Outgoing, GuardPredicates);
}
BasicBlock *llvm::CreateControlFlowHub(
DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
const BBSetVector &Incoming, const BBSetVector &Outgoing,
- const StringRef Prefix) {
+ const StringRef Prefix, Optional<unsigned> MaxControlFlowBooleans) {
if (Outgoing.size() < 2)
return Outgoing.front();
@@ -1779,7 +1844,7 @@ BasicBlock *llvm::CreateControlFlowHub(
SmallVector<WeakVH, 8> DeletionCandidates;
convertToGuardPredicates(GuardBlocks, DeletionCandidates, Incoming, Outgoing,
- Prefix);
+ Prefix, MaxControlFlowBooleans);
auto FirstGuardBlock = GuardBlocks.front();
// Update the PHINodes in each outgoing block to match the new control flow.
diff --git a/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
index 03140ef7feb1e..3be96ebc93a2a 100644
--- a/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
+++ b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
@@ -23,6 +23,7 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -30,6 +31,11 @@
using namespace llvm;
+static cl::opt<unsigned> MaxBooleansInControlFlowHub(
+ "max-booleans-in-control-flow-hub", cl::init(32), cl::Hidden,
+ cl::desc("Set the maximum number of outgoing blocks for using a boolean "
+ "value to record the exiting block in CreateControlFlowHub."));
+
namespace {
struct UnifyLoopExitsLegacyPass : public FunctionPass {
static char ID;
@@ -114,9 +120,9 @@ static void restoreSSA(const DominatorTree &DT, const Loop *L,
// didn't exist in the original CFG.
auto Def = II.first;
LLVM_DEBUG(dbgs() << "externally used: " << Def->getName() << "\n");
- auto NewPhi = PHINode::Create(Def->getType(), Incoming.size(),
- Def->getName() + ".moved",
- LoopExitBlock->getTerminator());
+ auto NewPhi =
+ PHINode::Create(Def->getType(), Incoming.size(),
+ Def->getName() + ".moved", &LoopExitBlock->front());
for (auto *In : Incoming) {
LLVM_DEBUG(dbgs() << "predecessor " << In->getName() << ": ");
if (Def->getParent() == In || DT.dominates(Def, In)) {
@@ -181,8 +187,9 @@ static bool unifyLoopExits(DominatorTree &DT, LoopInfo &LI, Loop *L) {
SmallVector<BasicBlock *, 8> GuardBlocks;
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
- auto LoopExitBlock = CreateControlFlowHub(&DTU, GuardBlocks, ExitingBlocks,
- Exits, "loop.exit");
+ auto LoopExitBlock =
+ CreateControlFlowHub(&DTU, GuardBlocks, ExitingBlocks, Exits, "loop.exit",
+ MaxBooleansInControlFlowHub.getValue());
restoreSSA(DT, L, ExitingBlocks, LoopExitBlock);
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
index 79a733fe5a879..695577a902e3f 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
@@ -185,8 +185,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32
; SI-NEXT: v_mov_b32_e32 v0, 3
; SI-NEXT: s_branch .LBB3_3
; SI-NEXT: .LBB3_1: ; in Loop: Header=BB3_3 Depth=1
-; SI-NEXT: s_mov_b64 s[10:11], -1
; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b64 s[10:11], -1
; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: .LBB3_2: ; %Flow
; SI-NEXT: ; in Loop: Header=BB3_3 Depth=1
@@ -194,8 +194,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32
; SI-NEXT: s_cbranch_vccnz .LBB3_8
; SI-NEXT: .LBB3_3: ; %while.cond
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: s_mov_b64 s[10:11], -1
+; SI-NEXT: s_mov_b64 s[8:9], -1
; SI-NEXT: s_mov_b64 s[12:13], -1
; SI-NEXT: s_mov_b64 vcc, s[0:1]
; SI-NEXT: s_cbranch_vccz .LBB3_2
@@ -260,8 +260,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32
; FLAT-NEXT: v_mov_b32_e32 v0, 3
; FLAT-NEXT: s_branch .LBB3_3
; FLAT-NEXT: .LBB3_1: ; in Loop: Header=BB3_3 Depth=1
-; FLAT-NEXT: s_mov_b64 s[10:11], -1
; FLAT-NEXT: s_mov_b64 s[8:9], 0
+; FLAT-NEXT: s_mov_b64 s[10:11], -1
; FLAT-NEXT: s_mov_b64 s[12:13], -1
; FLAT-NEXT: .LBB3_2: ; %Flow
; FLAT-NEXT: ; in Loop: Header=BB3_3 Depth=1
@@ -269,8 +269,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32
; FLAT-NEXT: s_cbranch_vccnz .LBB3_8
; FLAT-NEXT: .LBB3_3: ; %while.cond
; FLAT-NEXT: ; =>This Inner Loop Header: Depth=1
-; FLAT-NEXT: s_mov_b64 s[8:9], -1
; FLAT-NEXT: s_mov_b64 s[10:11], -1
+; FLAT-NEXT: s_mov_b64 s[8:9], -1
; FLAT-NEXT: s_mov_b64 s[12:13], -1
; FLAT-NEXT: s_mov_b64 vcc, s[0:1]
; FLAT-NEXT: s_cbranch_vccz .LBB3_2
diff --git a/llvm/test/Transforms/FixIrreducible/basic.ll b/llvm/test/Transforms/FixIrreducible/basic.ll
index fe34ac9b93bc7..7ba1360160db2 100644
--- a/llvm/test/Transforms/FixIrreducible/basic.ll
+++ b/llvm/test/Transforms/FixIrreducible/basic.ll
@@ -15,9 +15,9 @@ define i32 @basic(i1 %PredEntry, i1 %PredLeft, i1 %PredRight, i32 %X, i32 %Y) {
; CHECK-NEXT: [[Z:%.*]] = phi i32 [ [[L]], [[LEFT:%.*]] ], [ [[R_PHI_MOVED:%.*]], [[RIGHT:%.*]] ]
; CHECK-NEXT: ret i32 [[Z]]
; CHECK: irr.guard:
-; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY:%.*]] ], [ false, [[LEFT]] ]
+; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[L]], [[LEFT]] ]
; CHECK-NEXT: [[L_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[X:%.*]], [[ENTRY]] ], [ [[L_PHI_MOVED]], [[LEFT]] ]
-; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY]] ], [ [[L]], [[LEFT]] ]
+; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY]] ], [ false, [[LEFT]] ]
; CHECK-NEXT: br i1 [[GUARD_LEFT]], label [[LEFT]], label [[RIGHT]]
;
entry:
@@ -49,9 +49,9 @@ define i32 @feedback_loop(i1 %PredEntry, i1 %PredLeft, i1 %PredRight, i32 %X, i3
; CHECK-NEXT: [[Z:%.*]] = phi i32 [ [[L_PHI_MOVED:%.*]], [[LEFT:%.*]] ], [ [[R_PHI_MOVED:%.*]], [[RIGHT:%.*]] ]
; CHECK-NEXT: ret i32 [[Z]]
; CHECK: irr.guard:
-; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY:%.*]] ], [ false, [[LEFT]] ]
+; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[L_PHI_MOVED]], [[LEFT]] ]
; CHECK-NEXT: [[L_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[X:%.*]], [[ENTRY]] ], [ [[L_PHI_MOVED]], [[LEFT]] ]
-; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY]] ], [ [[L_PHI_MOVED]], [[LEFT]] ]
+; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY]] ], [ false, [[LEFT]] ]
; CHECK-NEXT: br i1 [[GUARD_LEFT]], label [[LEFT]], label [[RIGHT]]
;
entry:
@@ -89,9 +89,9 @@ define i32 @multiple_predecessors(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC
; CHECK-NEXT: [[RET:%.*]] = phi i32 [ [[C_PHI_MOVED:%.*]], [[C:%.*]] ], [ [[D_INC]], [[D:%.*]] ]
; CHECK-NEXT: ret i32 [[RET]]
; CHECK: irr.guard:
+; CHECK-NEXT: [[D_PHI_MOVED]] = phi i32 [ [[D_PHI_MOVED]], [[D]] ], [ [[Y:%.*]], [[B]] ], [ [[A_INC]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ]
+; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[Y]], [[B]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ]
; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[D]] ], [ [[PREDB_INV]], [[B]] ], [ [[PREDA:%.*]], [[A]] ], [ false, [[C]] ]
-; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[Y:%.*]], [[B]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ]
-; CHECK-NEXT: [[D_PHI_MOVED]] = phi i32 [ [[D_PHI_MOVED]], [[D]] ], [ [[Y]], [[B]] ], [ [[A_INC]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ]
; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[D]]
;
entry:
@@ -136,9 +136,9 @@ define i32 @separate_predecessors(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC
; CHECK-NEXT: [[RET:%.*]] = phi i32 [ [[C_PHI_MOVED:%.*]], [[C:%.*]] ], [ [[D_INC]], [[D:%.*]] ]
; CHECK-NEXT: ret i32 [[RET]]
; CHECK: irr.guard:
-; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[D]] ], [ true, [[A]] ], [ false, [[C]] ], [ false, [[B]] ]
-; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ], [ undef, [[B]] ]
; CHECK-NEXT: [[D_PHI_MOVED]] = phi i32 [ [[D_PHI_MOVED]], [[D]] ], [ undef, [[A]] ], [ [[C_PHI_MOVED]], [[C]] ], [ [[Y:%.*]], [[B]] ]
+; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ], [ undef, [[B]] ]
+; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[D]] ], [ true, [[A]] ], [ false, [[C]] ], [ false, [[B]] ]
; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[D]]
;
entry:
@@ -237,9 +237,9 @@ define i32 @hidden_nodes(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC, i1 %Pre
; CHECK: exit:
; CHECK-NEXT: ret i32 [[B_PHI_MOVED]]
; CHECK: irr.guard:
-; CHECK-NEXT: [[GUARD_A:%.*]] = phi i1 [ true, [[E]] ], [ [[PREDENTRY:%.*]], [[ENTRY:%.*]] ], [ false, [[A:%.*]] ]
+; CHECK-NEXT: [[B_PHI_MOVED]] = phi i32 [ undef, [[E]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[A_INC]], [[A:%.*]] ]
; CHECK-NEXT: [[A_PHI_MOVED]] = phi i32 [ [[C_INC]], [[E]] ], [ [[X:%.*]], [[ENTRY]] ], [ [[A_PHI_MOVED]], [[A]] ]
-; CHECK-NEXT: [[B_PHI_MOVED]] = phi i32 [ undef, [[E]] ], [ [[Y:%.*]], [[ENTRY]] ], [ [[A_INC]], [[A]] ]
+; CHECK-NEXT: [[GUARD_A:%.*]] = phi i1 [ true, [[E]] ], [ [[PREDENTRY:%.*]], [[ENTRY]] ], [ false, [[A]] ]
; CHECK-NEXT: br i1 [[GUARD_A]], label [[A]], label [[B:%.*]]
;
entry:
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
index b81f83a9e7094..f588d3fe7cef3 100644
--- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
@@ -56,12 +56,12 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3
; CHECK-NEXT: br i1 [[PRED5_INV]], label [[WHILE_COND47:%.*]], label [[FLOW10:%.*]]
; CHECK: Flow9:
; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ [[TMP22:%.*]], [[FLOW10]] ], [ undef, [[FLOW8]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ false, [[FLOW10]] ], [ undef, [[FLOW8]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ true, [[FLOW10]] ], [ [[TMP7]], [[FLOW8]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, [[FLOW10]] ], [ [[TMP8]], [[FLOW8]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ true, [[FLOW10]] ], [ [[TMP7]], [[FLOW8]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ false, [[FLOW10]] ], [ [[TMP8]], [[FLOW8]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, [[FLOW10]] ], [ undef, [[FLOW8]] ]
; CHECK-NEXT: [[TMP14:%.*]] = phi i1 [ [[TMP23:%.*]], [[FLOW10]] ], [ true, [[FLOW8]] ]
-; CHECK-NEXT: [[DOTINV11:%.*]] = xor i1 [[TMP12]], true
-; CHECK-NEXT: [[DOTINV:%.*]] = xor i1 [[TMP13]], true
+; CHECK-NEXT: [[DOTINV11:%.*]] = xor i1 [[TMP11]], true
+; CHECK-NEXT: [[DOTINV:%.*]] = xor i1 [[TMP12]], true
; CHECK-NEXT: br i1 [[TMP14]], label [[LOOP_EXIT_GUARD1:%.*]], label [[IRR_GUARD]]
; CHECK: while.cond47:
; CHECK-NEXT: br label [[FLOW10]]
@@ -111,7 +111,7 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3
; CHECK-NEXT: [[GUARD_COND_TRUE49_INV:%.*]] = xor i1 [[GUARD_COND_TRUE49]], true
; CHECK-NEXT: br i1 [[GUARD_COND_TRUE49_INV]], label [[COND_END61]], label [[FLOW7]]
; CHECK: Flow15:
-; CHECK-NEXT: [[TMP20]] = phi i1 [ false, [[IF_THEN69:%.*]] ], [ [[TMP11]], [[LOOP_EXIT_GUARD2:%.*]] ]
+; CHECK-NEXT: [[TMP20]] = phi i1 [ false, [[IF_THEN69:%.*]] ], [ [[TMP13]], [[LOOP_EXIT_GUARD2:%.*]] ]
; CHECK-NEXT: [[TMP21]] = phi i1 [ [[PRED8:%.*]], [[IF_THEN69]] ], [ [[DOTINV]], [[LOOP_EXIT_GUARD2]] ]
; CHECK-NEXT: br label [[FLOW14:%.*]]
; CHECK: loop.exit.guard:
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
index 6cc2689f08fd0..4b3a43000f17a 100644
--- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
@@ -36,8 +36,8 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
; CHECK: B2:
; CHECK-NEXT: br i1 [[PREDB2_INV]], label [[L2:%.*]], label [[FLOW2:%.*]]
; CHECK: Flow:
-; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, [[FLOW2]] ], [ undef, [[H2]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ false, [[FLOW2]] ], [ undef, [[H2]] ]
; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW2]] ], [ true, [[H2]] ]
; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_EXIT_GUARD1:%.*]], label [[H2]]
; CHECK: L2:
@@ -51,7 +51,7 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
; CHECK: exit:
; CHECK-NEXT: ret void
; CHECK: Flow5:
-; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ undef, [[L1:%.*]] ], [ [[TMP2]], [[LOOP_EXIT_GUARD1]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ undef, [[L1:%.*]] ], [ [[TMP3]], [[LOOP_EXIT_GUARD1]] ]
; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ false, [[L1]] ], [ true, [[LOOP_EXIT_GUARD1]] ]
; CHECK-NEXT: br label [[FLOW4]]
; CHECK: loop.exit.guard:
@@ -64,7 +64,7 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ [[TMP6]], [[FLOW5]] ], [ true, [[FLOW3]] ]
; CHECK-NEXT: br i1 [[TMP9]], label [[LOOP_EXIT_GUARD:%.*]], label [[H1]]
; CHECK: loop.exit.guard1:
-; CHECK-NEXT: br i1 [[TMP3]], label [[L1]], label [[FLOW5]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[L1]], label [[FLOW5]]
;
entry:
br label %H1
@@ -116,8 +116,8 @@ define void @incorrect-backedge(i1 %PredH2, i1 %PredH3, i1 %PredL2, i1 %PredL13,
; CHECK-NEXT: br i1 [[PREDL2_INV]], label [[L13:%.*]], label [[FLOW3:%.*]]
; CHECK: Flow:
; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ [[TMP7]], [[FLOW3]] ], [ false, [[H3]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[TMP8:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ [[TMP8:%.*]], [[FLOW3]] ], [ false, [[H3]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[TMP8]], [[FLOW3]] ], [ true, [[H3]] ]
; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ [[TMP9:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
; CHECK-NEXT: br i1 [[TMP3]], label [[LOOP_EXIT_GUARD2:%.*]], label [[H3]]
; CHECK: L13:
@@ -138,14 +138,14 @@ define void @incorrect-backedge(i1 %PredH2, i1 %PredH3, i1 %PredL2, i1 %PredL13,
; CHECK: loop.exit.guard1:
; CHECK-NEXT: br label [[FLOW5]]
; CHECK: Flow3:
-; CHECK-NEXT: [[TMP7]] = phi i1 [ false, [[L13]] ], [ undef, [[L2]] ]
-; CHECK-NEXT: [[TMP8]] = phi i1 [ true, [[L13]] ], [ false, [[L2]] ]
+; CHECK-NEXT: [[TMP7]] = phi i1 [ true, [[L13]] ], [ false, [[L2]] ]
+; CHECK-NEXT: [[TMP8]] = phi i1 [ false, [[L13]] ], [ undef, [[L2]] ]
; CHECK-NEXT: [[TMP9]] = phi i1 [ [[PREDL13_INV]], [[L13]] ], [ true, [[L2]] ]
; CHECK-NEXT: br label [[FLOW]]
; CHECK: Flow4:
-; CHECK-NEXT: [[TMP10]] = phi i1 [ [[TMP0]], [[LOOP_EXIT_GUARD2]] ], [ false, [[H2]] ]
+; CHECK-NEXT: [[TMP10]] = phi i1 [ [[TMP2]], [[LOOP_EXIT_GUARD2]] ], [ false, [[H2]] ]
; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ [[TMP1]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ [[TMP2]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ [[TMP0]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
; CHECK-NEXT: [[DOTINV]] = xor i1 [[TMP11]], true
; CHECK-NEXT: br i1 [[TMP12]], label [[LOOP_EXIT_GUARD]], label [[H2]]
; CHECK: loop.exit.guard2:
diff --git a/llvm/test/Transforms/UnifyLoopExits/integer_guards.ll b/llvm/test/Transforms/UnifyLoopExits/integer_guards.ll
new file mode 100644
index 0000000000000..803f14b24a976
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/integer_guards.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=unify-loop-exits -max-booleans-in-control-flow-hub=1 -S | FileCheck %s
+; RUN: opt < %s -passes=unify-loop-exits -S | FileCheck --check-prefix=BOOLEAN %s
+
+; A loop with multiple exit blocks.
+
+define void @loop_two_exits(i1 %PredEntry, i1 %PredA) {
+; CHECK-LABEL: @loop_two_exits(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[E:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[C:%.*]] ]
+; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C]]
+; CHECK: B:
+; CHECK-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT: br label [[D:%.*]]
+; CHECK: C:
+; CHECK-NEXT: [[INC2]] = add i32 [[INC1]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10
+; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: D:
+; CHECK-NEXT: unreachable
+; CHECK: E:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[MERGED_BB_IDX:%.*]] = phi i32 [ 0, [[A]] ], [ 1, [[C]] ]
+; CHECK-NEXT: [[B_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 0
+; CHECK-NEXT: br i1 [[B_PREDICATE]], label [[B:%.*]], label [[E]]
+;
+; BOOLEAN-LABEL: @loop_two_exits(
+; BOOLEAN-NEXT: entry:
+; BOOLEAN-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[E:%.*]]
+; BOOLEAN: A:
+; BOOLEAN-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[C:%.*]] ]
+; BOOLEAN-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C]]
+; BOOLEAN: B:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0:[0-9]+]]
+; BOOLEAN-NEXT: br label [[D:%.*]]
+; BOOLEAN: C:
+; BOOLEAN-NEXT: [[INC2]] = add i32 [[INC1]], 1
+; BOOLEAN-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10
+; BOOLEAN-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; BOOLEAN: D:
+; BOOLEAN-NEXT: unreachable
+; BOOLEAN: E:
+; BOOLEAN-NEXT: ret void
+; BOOLEAN: loop.exit.guard:
+; BOOLEAN-NEXT: [[GUARD_B:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; BOOLEAN-NEXT: br i1 [[GUARD_B]], label [[B:%.*]], label [[E]]
+;
+entry:
+ br i1 %PredEntry, label %A, label %E
+
+A:
+ %inc1 = phi i32 [ 0, %entry ], [ %inc2, %C ]
+ br i1 %PredA, label %B, label %C
+
+B:
+ tail call fastcc void @check(i32 1) #0
+ br label %D
+
+C:
+ %inc2 = add i32 %inc1, 1
+ %cmp = icmp ult i32 %inc2, 10
+ br i1 %cmp, label %A, label %E
+
+D:
+ unreachable
+
+E:
+ ret void
+}
+
+; The loop exit blocks appear in an inner loop.
+
+define void @inner_loop(i1 %PredEntry, i1 %PredA, i1 %PredB) {
+; CHECK-LABEL: @inner_loop(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[I:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[OUTER1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER2:%.*]], [[G:%.*]] ]
+; CHECK-NEXT: br label [[B:%.*]]
+; CHECK: B:
+; CHECK-NEXT: [[INNER1:%.*]] = phi i32 [ 0, [[A]] ], [ [[INNER2:%.*]], [[F:%.*]] ]
+; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]]
+; CHECK: C:
+; CHECK-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]]
+; CHECK-NEXT: br label [[H:%.*]]
+; CHECK: D:
+; CHECK-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD2]], label [[F]]
+; CHECK: E:
+; CHECK-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]]
+; CHECK-NEXT: br label [[H]]
+; CHECK: F:
+; CHECK-NEXT: [[INNER2]] = add i32 [[INNER1]], 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[INNER2]], 20
+; CHECK-NEXT: br i1 [[CMP1]], label [[B]], label [[LOOP_EXIT_GUARD2]]
+; CHECK: G:
+; CHECK-NEXT: [[OUTER2]] = add i32 [[OUTER1]], 1
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[OUTER2]], 10
+; CHECK-NEXT: br i1 [[CMP2]], label [[A]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK: H:
+; CHECK-NEXT: unreachable
+; CHECK: I:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[MERGED_BB_IDX:%.*]] = phi i32 [ 2, [[G]] ], [ [[MERGED_BB_IDX_MOVED:%.*]], [[LOOP_EXIT_GUARD2]] ]
+; CHECK-NEXT: [[C_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 0
+; CHECK-NEXT: br i1 [[C_PREDICATE]], label [[C:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; CHECK: loop.exit.guard1:
+; CHECK-NEXT: [[E_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 1
+; CHECK-NEXT: br i1 [[E_PREDICATE]], label [[E:%.*]], label [[I]]
+; CHECK: loop.exit.guard2:
+; CHECK-NEXT: [[MERGED_BB_IDX_MOVED]] = phi i32 [ 0, [[B]] ], [ 1, [[D]] ], [ undef, [[F]] ]
+; CHECK-NEXT: [[MERGED_BB_IDX3:%.*]] = phi i32 [ 0, [[B]] ], [ 0, [[D]] ], [ 1, [[F]] ]
+; CHECK-NEXT: [[LOOP_EXIT_GUARD_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX3]], 0
+; CHECK-NEXT: br i1 [[LOOP_EXIT_GUARD_PREDICATE]], label [[LOOP_EXIT_GUARD]], label [[G]]
+;
+; BOOLEAN-LABEL: @inner_loop(
+; BOOLEAN-NEXT: entry:
+; BOOLEAN-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[I:%.*]]
+; BOOLEAN: A:
+; BOOLEAN-NEXT: [[OUTER1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER2:%.*]], [[G:%.*]] ]
+; BOOLEAN-NEXT: br label [[B:%.*]]
+; BOOLEAN: B:
+; BOOLEAN-NEXT: [[INNER1:%.*]] = phi i32 [ 0, [[A]] ], [ [[INNER2:%.*]], [[F:%.*]] ]
+; BOOLEAN-NEXT: br i1 [[PREDA:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]]
+; BOOLEAN: C:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]]
+; BOOLEAN-NEXT: br label [[H:%.*]]
+; BOOLEAN: D:
+; BOOLEAN-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD2]], label [[F]]
+; BOOLEAN: E:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]]
+; BOOLEAN-NEXT: br label [[H]]
+; BOOLEAN: F:
+; BOOLEAN-NEXT: [[INNER2]] = add i32 [[INNER1]], 1
+; BOOLEAN-NEXT: [[CMP1:%.*]] = icmp ult i32 [[INNER2]], 20
+; BOOLEAN-NEXT: br i1 [[CMP1]], label [[B]], label [[LOOP_EXIT_GUARD2]]
+; BOOLEAN: G:
+; BOOLEAN-NEXT: [[OUTER2]] = add i32 [[OUTER1]], 1
+; BOOLEAN-NEXT: [[CMP2:%.*]] = icmp ult i32 [[OUTER2]], 10
+; BOOLEAN-NEXT: br i1 [[CMP2]], label [[A]], label [[LOOP_EXIT_GUARD:%.*]]
+; BOOLEAN: H:
+; BOOLEAN-NEXT: unreachable
+; BOOLEAN: I:
+; BOOLEAN-NEXT: ret void
+; BOOLEAN: loop.exit.guard:
+; BOOLEAN-NEXT: [[GUARD_C:%.*]] = phi i1 [ false, [[G]] ], [ [[GUARD_C_MOVED:%.*]], [[LOOP_EXIT_GUARD2]] ]
+; BOOLEAN-NEXT: [[GUARD_E:%.*]] = phi i1 [ false, [[G]] ], [ [[GUARD_E_MOVED:%.*]], [[LOOP_EXIT_GUARD2]] ]
+; BOOLEAN-NEXT: br i1 [[GUARD_C]], label [[C:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; BOOLEAN: loop.exit.guard1:
+; BOOLEAN-NEXT: br i1 [[GUARD_E]], label [[E:%.*]], label [[I]]
+; BOOLEAN: loop.exit.guard2:
+; BOOLEAN-NEXT: [[GUARD_E_MOVED]] = phi i1 [ false, [[B]] ], [ true, [[D]] ], [ undef, [[F]] ]
+; BOOLEAN-NEXT: [[GUARD_C_MOVED]] = phi i1 [ true, [[B]] ], [ false, [[D]] ], [ undef, [[F]] ]
+; BOOLEAN-NEXT: [[GUARD_LOOP_EXIT_GUARD:%.*]] = phi i1 [ true, [[B]] ], [ true, [[D]] ], [ false, [[F]] ]
+; BOOLEAN-NEXT: br i1 [[GUARD_LOOP_EXIT_GUARD]], label [[LOOP_EXIT_GUARD]], label [[G]]
+;
+entry:
+ br i1 %PredEntry, label %A, label %I
+
+A:
+ %outer1 = phi i32 [ 0, %entry ], [ %outer2, %G ]
+ br label %B
+
+B:
+ %inner1 = phi i32 [ 0, %A ], [ %inner2, %F ]
+ br i1 %PredA, label %D, label %C
+
+C:
+ tail call fastcc void @check(i32 1) #0
+ br label %H
+
+D:
+ br i1 %PredB, label %E, label %F
+
+E:
+ tail call fastcc void @check(i32 2) #0
+ br label %H
+
+F:
+ %inner2 = add i32 %inner1, 1
+ %cmp1 = icmp ult i32 %inner2, 20
+ br i1 %cmp1, label %B, label %G
+
+G:
+ %outer2 = add i32 %outer1, 1
+ %cmp2 = icmp ult i32 %outer2, 10
+ br i1 %cmp2, label %A, label %I
+
+H:
+ unreachable
+
+I:
+ ret void
+}
+
+; A loop with more exit blocks.
+
+define void @loop_five_exits(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC, i1 %PredD) {
+; CHECK-LABEL: @loop_five_exits(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[L:%.*]]
+; CHECK: A:
+; CHECK-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[I:%.*]] ]
+; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C:%.*]]
+; CHECK: B:
+; CHECK-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]]
+; CHECK-NEXT: br label [[J:%.*]]
+; CHECK: C:
+; CHECK-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD]], label [[E:%.*]]
+; CHECK: D:
+; CHECK-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]]
+; CHECK-NEXT: br label [[J]]
+; CHECK: E:
+; CHECK-NEXT: br i1 [[PREDC:%.*]], label [[LOOP_EXIT_GUARD]], label [[G:%.*]]
+; CHECK: F:
+; CHECK-NEXT: tail call fastcc void @check(i32 3) #[[ATTR0]]
+; CHECK-NEXT: br label [[K:%.*]]
+; CHECK: G:
+; CHECK-NEXT: br i1 [[PREDD:%.*]], label [[LOOP_EXIT_GUARD]], label [[I]]
+; CHECK: H:
+; CHECK-NEXT: tail call fastcc void @check(i32 4) #[[ATTR0]]
+; CHECK-NEXT: br label [[K]]
+; CHECK: I:
+; CHECK-NEXT: [[INC2]] = add i32 [[INC1]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10
+; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK: J:
+; CHECK-NEXT: br label [[L]]
+; CHECK: K:
+; CHECK-NEXT: br label [[L]]
+; CHECK: L:
+; CHECK-NEXT: ret void
+; CHECK: loop.exit.guard:
+; CHECK-NEXT: [[MERGED_BB_IDX:%.*]] = phi i32 [ 0, [[A]] ], [ 1, [[C]] ], [ 2, [[E]] ], [ 3, [[G]] ], [ 4, [[I]] ]
+; CHECK-NEXT: [[B_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 0
+; CHECK-NEXT: br i1 [[B_PREDICATE]], label [[B:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; CHECK: loop.exit.guard1:
+; CHECK-NEXT: [[D_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 1
+; CHECK-NEXT: br i1 [[D_PREDICATE]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]]
+; CHECK: loop.exit.guard2:
+; CHECK-NEXT: [[F_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 2
+; CHECK-NEXT: br i1 [[F_PREDICATE]], label [[F:%.*]], label [[LOOP_EXIT_GUARD3:%.*]]
+; CHECK: loop.exit.guard3:
+; CHECK-NEXT: [[H_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 3
+; CHECK-NEXT: br i1 [[H_PREDICATE]], label [[H:%.*]], label [[L]]
+;
+; BOOLEAN-LABEL: @loop_five_exits(
+; BOOLEAN-NEXT: entry:
+; BOOLEAN-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[L:%.*]]
+; BOOLEAN: A:
+; BOOLEAN-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[I:%.*]] ]
+; BOOLEAN-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C:%.*]]
+; BOOLEAN: B:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]]
+; BOOLEAN-NEXT: br label [[J:%.*]]
+; BOOLEAN: C:
+; BOOLEAN-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD]], label [[E:%.*]]
+; BOOLEAN: D:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]]
+; BOOLEAN-NEXT: br label [[J]]
+; BOOLEAN: E:
+; BOOLEAN-NEXT: br i1 [[PREDC:%.*]], label [[LOOP_EXIT_GUARD]], label [[G:%.*]]
+; BOOLEAN: F:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 3) #[[ATTR0]]
+; BOOLEAN-NEXT: br label [[K:%.*]]
+; BOOLEAN: G:
+; BOOLEAN-NEXT: br i1 [[PREDD:%.*]], label [[LOOP_EXIT_GUARD]], label [[I]]
+; BOOLEAN: H:
+; BOOLEAN-NEXT: tail call fastcc void @check(i32 4) #[[ATTR0]]
+; BOOLEAN-NEXT: br label [[K]]
+; BOOLEAN: I:
+; BOOLEAN-NEXT: [[INC2]] = add i32 [[INC1]], 1
+; BOOLEAN-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10
+; BOOLEAN-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; BOOLEAN: J:
+; BOOLEAN-NEXT: br label [[L]]
+; BOOLEAN: K:
+; BOOLEAN-NEXT: br label [[L]]
+; BOOLEAN: L:
+; BOOLEAN-NEXT: ret void
+; BOOLEAN: loop.exit.guard:
+; BOOLEAN-NEXT: [[GUARD_B:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ], [ false, [[E]] ], [ false, [[G]] ], [ false, [[I]] ]
+; BOOLEAN-NEXT: [[GUARD_D:%.*]] = phi i1 [ false, [[A]] ], [ true, [[C]] ], [ false, [[E]] ], [ false, [[G]] ], [ false, [[I]] ]
+; BOOLEAN-NEXT: [[GUARD_F:%.*]] = phi i1 [ false, [[A]] ], [ false, [[C]] ], [ true, [[E]] ], [ false, [[G]] ], [ false, [[I]] ]
+; BOOLEAN-NEXT: [[GUARD_H:%.*]] = phi i1 [ false, [[A]] ], [ false, [[C]] ], [ false, [[E]] ], [ true, [[G]] ], [ false, [[I]] ]
+; BOOLEAN-NEXT: br i1 [[GUARD_B]], label [[B:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; BOOLEAN: loop.exit.guard1:
+; BOOLEAN-NEXT: br i1 [[GUARD_D]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]]
+; BOOLEAN: loop.exit.guard2:
+; BOOLEAN-NEXT: br i1 [[GUARD_F]], label [[F:%.*]], label [[LOOP_EXIT_GUARD3:%.*]]
+; BOOLEAN: loop.exit.guard3:
+; BOOLEAN-NEXT: br i1 [[GUARD_H]], label [[H:%.*]], label [[L]]
+;
+entry:
+ br i1 %PredEntry, label %A, label %L
+
+A:
+ %inc1 = phi i32 [ 0, %entry ], [ %inc2, %I ]
+ br i1 %PredA, label %B, label %C
+
+B:
+ tail call fastcc void @check(i32 1) #0
+ br label %J
+
+C:
+ br i1 %PredB, label %D, label %E
+
+D:
+ tail call fastcc void @check(i32 2) #0
+ br label %J
+
+E:
+ br i1 %PredC, label %F, label %G
+
+F:
+ tail call fastcc void @check(i32 3) #0
+ br label %K
+
+G:
+ br i1 %PredD, label %H, label %I
+
+H:
+ tail call fastcc void @check(i32 4) #0
+ br label %K
+
+I:
+ %inc2 = add i32 %inc1, 1
+ %cmp = icmp ult i32 %inc2, 10
+ br i1 %cmp, label %A, label %L
+
+J:
+ br label %L
+
+K:
+ br label %L
+
+L:
+ ret void
+}
+
+
+declare void @check(i32 noundef %i) #0
+
+attributes #0 = { noreturn nounwind }
+
diff --git a/llvm/test/Transforms/UnifyLoopExits/nested.ll b/llvm/test/Transforms/UnifyLoopExits/nested.ll
index 729e51db57303..9973aa305cdf1 100644
--- a/llvm/test/Transforms/UnifyLoopExits/nested.ll
+++ b/llvm/test/Transforms/UnifyLoopExits/nested.ll
@@ -31,12 +31,12 @@ define void @nested(i1 %PredB3, i1 %PredB4, i1 %PredA4, i1 %PredA3, i32 %X, i32
; CHECK-NEXT: [[EXIT_PHI:%.*]] = phi i32 [ [[Z:%.*]], [[C:%.*]] ], [ [[EXIT_PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
; CHECK-NEXT: ret void
; CHECK: loop.exit.guard:
-; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[A4]] ], [ false, [[A5]] ]
; CHECK-NEXT: [[EXIT_PHI_MOVED]] = phi i32 [ undef, [[A4]] ], [ [[A4_PHI]], [[A5]] ]
+; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[A4]] ], [ false, [[A5]] ]
; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[EXIT]]
; CHECK: loop.exit.guard1:
-; CHECK-NEXT: [[GUARD_A3:%.*]] = phi i1 [ true, [[B3]] ], [ false, [[B4]] ]
; CHECK-NEXT: [[X_INC_MOVED]] = phi i32 [ [[X_INC]], [[B3]] ], [ [[X_INC]], [[B4]] ]
+; CHECK-NEXT: [[GUARD_A3:%.*]] = phi i1 [ true, [[B3]] ], [ false, [[B4]] ]
; CHECK-NEXT: br i1 [[GUARD_A3]], label [[A3]], label [[A2]]
;
entry:
diff --git a/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
index 5209979aa725b..eb42858cbbf33 100644
--- a/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
+++ b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
@@ -30,9 +30,9 @@ define i32 @exiting-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr alig
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[INC]], [[C:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
; CHECK-NEXT: ret i32 [[PHI]]
; CHECK: loop.exit.guard:
-; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ]
-; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP42]], [[A]] ], [ undef, [[B]] ]
; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[B]] ]
+; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP42]], [[A]] ], [ undef, [[B]] ]
+; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ]
; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[C]]
;
entry:
@@ -84,8 +84,8 @@ define i32 @internal-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr ali
; CHECK: return:
; CHECK-NEXT: ret i32 0
; CHECK: loop.exit.guard:
-; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D:%.*]]
;
entry:
@@ -141,9 +141,9 @@ define i32 @mixed-use-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[ENTRY:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
; CHECK-NEXT: ret i32 [[PHI]]
; CHECK: loop.exit.guard:
-; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
-; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP43]], [[A]] ], [ undef, [[C]] ]
; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP43]], [[A]] ], [ undef, [[C]] ]
+; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D]]
;
entry:
@@ -206,8 +206,8 @@ define i32 @phi-via-external-block(i32* %arg1, i32* %arg2) local_unnamed_addr al
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[E:%.*]] ]
; CHECK-NEXT: ret i32 [[PHI]]
; CHECK: loop.exit.guard:
-; CHECK-NEXT: [[GUARD_E:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT: [[GUARD_E:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
; CHECK-NEXT: br i1 [[GUARD_E]], label [[E]], label [[D]]
;
entry:
More information about the llvm-commits
mailing list