[llvm] [NFC] Extract LoopConstrainer from IRCE to reuse it outside the pass (PR #70508)
Aleksandr Popov via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 31 08:05:39 PDT 2023
https://github.com/aleks-tmb updated https://github.com/llvm/llvm-project/pull/70508
>From 81027351eb4c9afc39f08326ca3d39a0bc52acb7 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 09:38:00 -0700
Subject: [PATCH 1/8] [NFC][IRCE] Move ExitCountTy form LoopConstrainer to
LoopStructure
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
Move ExitCountTy out of the LoopConstrainer to avoid MaxBETakenCount
recalculation.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index bf40b355651c3ab..c8166565136b184 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -616,6 +616,7 @@ struct LoopStructure {
Value *LoopExitAt = nullptr;
bool IndVarIncreasing = false;
bool IsSignedPredicate = true;
+ IntegerType *ExitCountTy = nullptr;
LoopStructure() = default;
@@ -633,6 +634,7 @@ struct LoopStructure {
Result.LoopExitAt = Map(LoopExitAt);
Result.IndVarIncreasing = IndVarIncreasing;
Result.IsSignedPredicate = IsSignedPredicate;
+ Result.ExitCountTy = ExitCountTy;
return Result;
}
@@ -756,7 +758,6 @@ class LoopConstrainer {
// Information about the original loop we started out with.
Loop &OriginalLoop;
- const IntegerType *ExitCountTy = nullptr;
BasicBlock *OriginalPreheader = nullptr;
// The preheader of the main loop. This may or may not be different from
@@ -1191,6 +1192,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
Result.IndVarIncreasing = IsIncreasing;
Result.LoopExitAt = RightValue;
Result.IsSignedPredicate = IsSignedPredicate;
+ Result.ExitCountTy = cast<IntegerType>(MaxBETakenCount->getType());
FailureReason = nullptr;
@@ -1208,9 +1210,9 @@ std::optional<LoopConstrainer::SubRanges>
LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
auto *RTy = cast<IntegerType>(Range.getType());
// We only support wide range checks and narrow latches.
- if (!AllowNarrowLatchCondition && RTy != ExitCountTy)
+ if (!AllowNarrowLatchCondition && RTy != MainLoopStructure.ExitCountTy)
return std::nullopt;
- if (RTy->getBitWidth() < ExitCountTy->getBitWidth())
+ if (RTy->getBitWidth() < MainLoopStructure.ExitCountTy->getBitWidth())
return std::nullopt;
LoopConstrainer::SubRanges Result;
@@ -1543,13 +1545,8 @@ Loop *LoopConstrainer::createClonedLoopStructure(Loop *Original, Loop *Parent,
}
bool LoopConstrainer::run() {
- BasicBlock *Preheader = nullptr;
- const SCEV *MaxBETakenCount =
- getNarrowestLatchMaxTakenCountEstimate(SE, OriginalLoop);
- Preheader = OriginalLoop.getLoopPreheader();
- assert(!isa<SCEVCouldNotCompute>(MaxBETakenCount) && Preheader != nullptr &&
- "preconditions!");
- ExitCountTy = cast<IntegerType>(MaxBETakenCount->getType());
+ BasicBlock *Preheader = OriginalLoop.getLoopPreheader();
+ assert(Preheader != nullptr && "precondition!");
OriginalPreheader = Preheader;
MainLoopPreheader = Preheader;
>From efb10c94d7b491be367465508409a731a02f5a1a Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 10:32:09 -0700
Subject: [PATCH 2/8] [NFC][IRCE] Pass AllowUnsignedLatchCondition as a
parameter to parseLoopStructure
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
Remove LoopStructure's dependency on the AllowUnsignedLatchCondition option.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index c8166565136b184..c2e44bbe44d843b 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -638,8 +638,8 @@ struct LoopStructure {
return Result;
}
- static std::optional<LoopStructure> parseLoopStructure(ScalarEvolution &,
- Loop &, const char *&);
+ static std::optional<LoopStructure>
+ parseLoopStructure(ScalarEvolution &, Loop &, bool, const char *&);
};
/// This class is used to constrain loops to run within a given iteration space.
@@ -894,6 +894,7 @@ static const SCEV *getNarrowestLatchMaxTakenCountEstimate(ScalarEvolution &SE,
std::optional<LoopStructure>
LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
+ bool AllowUnsignedLatchCond,
const char *&FailureReason) {
if (!L.isLoopSimplifyForm()) {
FailureReason = "loop not in LoopSimplify form";
@@ -1077,7 +1078,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
}
IsSignedPredicate = ICmpInst::isSigned(Pred);
- if (!IsSignedPredicate && !AllowUnsignedLatchCondition) {
+ if (!IsSignedPredicate && !AllowUnsignedLatchCond) {
FailureReason = "unsigned latch conditions are explicitly prohibited";
return std::nullopt;
}
@@ -1142,7 +1143,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
IsSignedPredicate =
Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT;
- if (!IsSignedPredicate && !AllowUnsignedLatchCondition) {
+ if (!IsSignedPredicate && !AllowUnsignedLatchCond) {
FailureReason = "unsigned latch conditions are explicitly prohibited";
return std::nullopt;
}
@@ -2106,7 +2107,8 @@ bool InductiveRangeCheckElimination::run(
const char *FailureReason = nullptr;
std::optional<LoopStructure> MaybeLoopStructure =
- LoopStructure::parseLoopStructure(SE, *L, FailureReason);
+ LoopStructure::parseLoopStructure(SE, *L, AllowUnsignedLatchCondition,
+ FailureReason);
if (!MaybeLoopStructure) {
LLVM_DEBUG(dbgs() << "irce: could not parse loop structure: "
<< FailureReason << "\n";);
>From 68ccec5eb9483ec946137af8daeaea89ff469be1 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 11:51:06 -0700
Subject: [PATCH 3/8] [NFC][IRCE] Move calculateSubRanges from LoopConstrainer
to IRCE
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
Move the specific for IRCE calculateSubRanges method outside of
LoopConstrainer.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 67 ++++++++++---------
1 file changed, 36 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index c2e44bbe44d843b..415ecc6e8fc1daf 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -651,6 +651,20 @@ struct LoopStructure {
/// which the induction variable is < Begin, and the post loop runs any
/// iterations in which the induction variable is >= End.
class LoopConstrainer {
+public:
+ // Calculated subranges we restrict the iteration space of the main loop to.
+ // See the implementation of `calculateSubRanges' for more details on how
+ // these fields are computed. `LowLimit` is std::nullopt if there is no
+ // restriction on low end of the restricted iteration space of the main loop.
+ // `HighLimit` is std::nullopt if there is no restriction on high end of the
+ // restricted iteration space of the main loop.
+
+ struct SubRanges {
+ std::optional<const SCEV *> LowLimit;
+ std::optional<const SCEV *> HighLimit;
+ };
+
+private:
// The representation of a clone of the original loop we started out with.
struct ClonedLoop {
// The cloned blocks
@@ -674,23 +688,6 @@ class LoopConstrainer {
RewrittenRangeInfo() = default;
};
- // Calculated subranges we restrict the iteration space of the main loop to.
- // See the implementation of `calculateSubRanges' for more details on how
- // these fields are computed. `LowLimit` is std::nullopt if there is no
- // restriction on low end of the restricted iteration space of the main loop.
- // `HighLimit` is std::nullopt if there is no restriction on high end of the
- // restricted iteration space of the main loop.
-
- struct SubRanges {
- std::optional<const SCEV *> LowLimit;
- std::optional<const SCEV *> HighLimit;
- };
-
- // Compute a safe set of limits for the main loop to run in -- effectively the
- // intersection of `Range' and the iteration space of the original loop.
- // Return std::nullopt if unable to compute the set of subranges.
- std::optional<SubRanges> calculateSubRanges(bool IsSignedPredicate) const;
-
// Clone `OriginalLoop' and return the result in CLResult. The IR after
// running `cloneLoop' is well formed except for the PHI nodes in CLResult --
// the PHI nodes say that there is an incoming edge from `OriginalPreheader`
@@ -771,14 +768,16 @@ class LoopConstrainer {
// for a definition)
LoopStructure MainLoopStructure;
+ SubRanges SR;
+
public:
LoopConstrainer(Loop &L, LoopInfo &LI,
function_ref<void(Loop *, bool)> LPMAddNewLoop,
const LoopStructure &LS, ScalarEvolution &SE,
- DominatorTree &DT, InductiveRangeCheck::Range R)
+ DominatorTree &DT, InductiveRangeCheck::Range R, SubRanges SR)
: F(*L.getHeader()->getParent()), Ctx(L.getHeader()->getContext()),
SE(SE), DT(DT), LI(LI), LPMAddNewLoop(LPMAddNewLoop), OriginalLoop(L),
- Range(R), MainLoopStructure(LS) {}
+ Range(R), MainLoopStructure(LS), SR(SR) {}
// Entry point for the algorithm. Returns true on success.
bool run();
@@ -1207,8 +1206,13 @@ static const SCEV *NoopOrExtend(const SCEV *S, Type *Ty, ScalarEvolution &SE,
return Signed ? SE.getNoopOrSignExtend(S, Ty) : SE.getNoopOrZeroExtend(S, Ty);
}
-std::optional<LoopConstrainer::SubRanges>
-LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
+// Compute a safe set of limits for the main loop to run in -- effectively the
+// intersection of `Range' and the iteration space of the original loop.
+// Return std::nullopt if unable to compute the set of subranges.
+static std::optional<LoopConstrainer::SubRanges>
+calculateSubRanges(ScalarEvolution &SE, const Loop &L,
+ InductiveRangeCheck::Range &Range,
+ const LoopStructure &MainLoopStructure) {
auto *RTy = cast<IntegerType>(Range.getType());
// We only support wide range checks and narrow latches.
if (!AllowNarrowLatchCondition && RTy != MainLoopStructure.ExitCountTy)
@@ -1218,6 +1222,7 @@ LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
LoopConstrainer::SubRanges Result;
+ bool IsSignedPredicate = MainLoopStructure.IsSignedPredicate;
// I think we can be more aggressive here and make this nuw / nsw if the
// addition that feeds into the icmp for the latch's terminating branch is nuw
// / nsw. In any case, a wrapping 2's complement addition is safe.
@@ -1261,7 +1266,7 @@ LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
GreatestSeen = Start;
}
- auto Clamp = [this, Smallest, Greatest, IsSignedPredicate](const SCEV *S) {
+ auto Clamp = [&SE, Smallest, Greatest, IsSignedPredicate](const SCEV *S) {
return IsSignedPredicate
? SE.getSMaxExpr(Smallest, SE.getSMinExpr(Greatest, S))
: SE.getUMaxExpr(Smallest, SE.getUMinExpr(Greatest, S));
@@ -1551,15 +1556,7 @@ bool LoopConstrainer::run() {
OriginalPreheader = Preheader;
MainLoopPreheader = Preheader;
-
bool IsSignedPredicate = MainLoopStructure.IsSignedPredicate;
- std::optional<SubRanges> MaybeSR = calculateSubRanges(IsSignedPredicate);
- if (!MaybeSR) {
- LLVM_DEBUG(dbgs() << "irce: could not compute subranges\n");
- return false;
- }
-
- SubRanges SR = *MaybeSR;
bool Increasing = MainLoopStructure.IndVarIncreasing;
IntegerType *IVTy =
cast<IntegerType>(Range.getBegin()->getType());
@@ -2147,7 +2144,15 @@ bool InductiveRangeCheckElimination::run(
if (!SafeIterRange)
return Changed;
- LoopConstrainer LC(*L, LI, LPMAddNewLoop, LS, SE, DT, *SafeIterRange);
+ std::optional<LoopConstrainer::SubRanges> MaybeSR =
+ calculateSubRanges(SE, *L, *SafeIterRange, LS);
+ if (!MaybeSR) {
+ LLVM_DEBUG(dbgs() << "irce: could not compute subranges\n");
+ return false;
+ }
+
+ LoopConstrainer LC(*L, LI, LPMAddNewLoop, LS, SE, DT, *SafeIterRange,
+ *MaybeSR);
if (LC.run()) {
Changed = true;
>From 73ade600ed2d16e1ad7df8d9081f1515af1c219b Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 12:39:48 -0700
Subject: [PATCH 4/8] [NFC] Remove dependency on InductiveRangeCheck from
LoopConstrainer
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
Replace InductiveRangeCheck::Range typed field by Type* one.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 16 +++++++---------
1 file changed, 7 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 415ecc6e8fc1daf..f57840a684925ef 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -761,8 +761,8 @@ class LoopConstrainer {
// `OriginalPreheader'.
BasicBlock *MainLoopPreheader = nullptr;
- // The range we need to run the main loop in.
- InductiveRangeCheck::Range Range;
+ // Type of the range we need to run the main loop in.
+ Type *RangeTy;
// The structure of the main loop (see comment at the beginning of this class
// for a definition)
@@ -774,10 +774,10 @@ class LoopConstrainer {
LoopConstrainer(Loop &L, LoopInfo &LI,
function_ref<void(Loop *, bool)> LPMAddNewLoop,
const LoopStructure &LS, ScalarEvolution &SE,
- DominatorTree &DT, InductiveRangeCheck::Range R, SubRanges SR)
+ DominatorTree &DT, Type *T, SubRanges SR)
: F(*L.getHeader()->getParent()), Ctx(L.getHeader()->getContext()),
SE(SE), DT(DT), LI(LI), LPMAddNewLoop(LPMAddNewLoop), OriginalLoop(L),
- Range(R), MainLoopStructure(LS), SR(SR) {}
+ RangeTy(T), MainLoopStructure(LS), SR(SR) {}
// Entry point for the algorithm. Returns true on success.
bool run();
@@ -1429,7 +1429,6 @@ LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
bool IsSignedPredicate = LS.IsSignedPredicate;
IRBuilder<> B(PreheaderJump);
- auto *RangeTy = Range.getBegin()->getType();
auto NoopOrExt = [&](Value *V) {
if (V->getType() == RangeTy)
return V;
@@ -1558,8 +1557,7 @@ bool LoopConstrainer::run() {
MainLoopPreheader = Preheader;
bool IsSignedPredicate = MainLoopStructure.IsSignedPredicate;
bool Increasing = MainLoopStructure.IndVarIncreasing;
- IntegerType *IVTy =
- cast<IntegerType>(Range.getBegin()->getType());
+ IntegerType *IVTy = cast<IntegerType>(RangeTy);
SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "irce");
Instruction *InsertPt = OriginalPreheader->getTerminator();
@@ -2151,8 +2149,8 @@ bool InductiveRangeCheckElimination::run(
return false;
}
- LoopConstrainer LC(*L, LI, LPMAddNewLoop, LS, SE, DT, *SafeIterRange,
- *MaybeSR);
+ LoopConstrainer LC(*L, LI, LPMAddNewLoop, LS, SE, DT,
+ SafeIterRange->getBegin()->getType(), *MaybeSR);
if (LC.run()) {
Changed = true;
>From 84c850b11eaf5cf4d5fd8fd399f1aa58565e4074 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 10:41:50 -0700
Subject: [PATCH 5/8] [NCF] Remove irce mention from LoopConstrainer logs
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 36 +++++++++----------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index f57840a684925ef..f907dcc023f295a 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -801,12 +801,12 @@ static bool isSafeDecreasingBound(const SCEV *Start,
assert(SE.isKnownNegative(Step) && "expecting negative step");
- LLVM_DEBUG(dbgs() << "irce: isSafeDecreasingBound with:\n");
- LLVM_DEBUG(dbgs() << "irce: Start: " << *Start << "\n");
- LLVM_DEBUG(dbgs() << "irce: Step: " << *Step << "\n");
- LLVM_DEBUG(dbgs() << "irce: BoundSCEV: " << *BoundSCEV << "\n");
- LLVM_DEBUG(dbgs() << "irce: Pred: " << Pred << "\n");
- LLVM_DEBUG(dbgs() << "irce: LatchExitBrIdx: " << LatchBrExitIdx << "\n");
+ LLVM_DEBUG(dbgs() << "isSafeDecreasingBound with:\n");
+ LLVM_DEBUG(dbgs() << "Start: " << *Start << "\n");
+ LLVM_DEBUG(dbgs() << "Step: " << *Step << "\n");
+ LLVM_DEBUG(dbgs() << "BoundSCEV: " << *BoundSCEV << "\n");
+ LLVM_DEBUG(dbgs() << "Pred: " << Pred << "\n");
+ LLVM_DEBUG(dbgs() << "LatchExitBrIdx: " << LatchBrExitIdx << "\n");
bool IsSigned = ICmpInst::isSigned(Pred);
// The predicate that we need to check that the induction variable lies
@@ -848,12 +848,12 @@ static bool isSafeIncreasingBound(const SCEV *Start,
if (!SE.isAvailableAtLoopEntry(BoundSCEV, L))
return false;
- LLVM_DEBUG(dbgs() << "irce: isSafeIncreasingBound with:\n");
- LLVM_DEBUG(dbgs() << "irce: Start: " << *Start << "\n");
- LLVM_DEBUG(dbgs() << "irce: Step: " << *Step << "\n");
- LLVM_DEBUG(dbgs() << "irce: BoundSCEV: " << *BoundSCEV << "\n");
- LLVM_DEBUG(dbgs() << "irce: Pred: " << Pred << "\n");
- LLVM_DEBUG(dbgs() << "irce: LatchExitBrIdx: " << LatchBrExitIdx << "\n");
+ LLVM_DEBUG(dbgs() << "isSafeIncreasingBound with:\n");
+ LLVM_DEBUG(dbgs() << "Start: " << *Start << "\n");
+ LLVM_DEBUG(dbgs() << "Step: " << *Step << "\n");
+ LLVM_DEBUG(dbgs() << "BoundSCEV: " << *BoundSCEV << "\n");
+ LLVM_DEBUG(dbgs() << "Pred: " << Pred << "\n");
+ LLVM_DEBUG(dbgs() << "LatchExitBrIdx: " << LatchBrExitIdx << "\n");
bool IsSigned = ICmpInst::isSigned(Pred);
// The predicate that we need to check that the induction variable lies
@@ -1168,7 +1168,7 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
assert(!L.contains(LatchExit) && "expected an exit block!");
const DataLayout &DL = Preheader->getModule()->getDataLayout();
- SCEVExpander Expander(SE, DL, "irce");
+ SCEVExpander Expander(SE, DL, "loop-constrainer");
Instruction *Ins = Preheader->getTerminator();
if (FixedRightSCEV)
@@ -1559,7 +1559,7 @@ bool LoopConstrainer::run() {
bool Increasing = MainLoopStructure.IndVarIncreasing;
IntegerType *IVTy = cast<IntegerType>(RangeTy);
- SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "irce");
+ SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "loop-constrainer");
Instruction *InsertPt = OriginalPreheader->getTerminator();
// It would have been better to make `PreLoop' and `PostLoop'
@@ -1585,14 +1585,14 @@ bool LoopConstrainer::run() {
IsSignedPredicate))
ExitPreLoopAtSCEV = SE.getAddExpr(*SR.HighLimit, MinusOneS);
else {
- LLVM_DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
+ LLVM_DEBUG(dbgs() << "could not prove no-overflow when computing "
<< "preloop exit limit. HighLimit = "
<< *(*SR.HighLimit) << "\n");
return false;
}
if (!Expander.isSafeToExpandAt(ExitPreLoopAtSCEV, InsertPt)) {
- LLVM_DEBUG(dbgs() << "irce: could not prove that it is safe to expand the"
+ LLVM_DEBUG(dbgs() << "could not prove that it is safe to expand the"
<< " preloop exit limit " << *ExitPreLoopAtSCEV
<< " at block " << InsertPt->getParent()->getName()
<< "\n");
@@ -1612,14 +1612,14 @@ bool LoopConstrainer::run() {
IsSignedPredicate))
ExitMainLoopAtSCEV = SE.getAddExpr(*SR.LowLimit, MinusOneS);
else {
- LLVM_DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
+ LLVM_DEBUG(dbgs() << "could not prove no-overflow when computing "
<< "mainloop exit limit. LowLimit = "
<< *(*SR.LowLimit) << "\n");
return false;
}
if (!Expander.isSafeToExpandAt(ExitMainLoopAtSCEV, InsertPt)) {
- LLVM_DEBUG(dbgs() << "irce: could not prove that it is safe to expand the"
+ LLVM_DEBUG(dbgs() << "could not prove that it is safe to expand the"
<< " main loop exit limit " << *ExitMainLoopAtSCEV
<< " at block " << InsertPt->getParent()->getName()
<< "\n");
>From fc98921b7a5b9eaf1aafc0a7fec154fad3d51890 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 10:46:50 -0700
Subject: [PATCH 6/8] [NFC] Rename 'irce.loop.clone' metadata to
'loop_constrainer.loop.clone'
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
---
.../Scalar/InductiveRangeCheckElimination.cpp | 2 +-
.../IRCE/add-metadata-pre-post-loops.ll | 6 ++---
.../Transforms/IRCE/conjunctive-checks.ll | 4 ++--
.../test/Transforms/IRCE/correct-loop-info.ll | 4 ++--
.../IRCE/iv-plus-offset-range-check.ll | 18 +++++++-------
.../IRCE/multiple-access-no-preloop.ll | 2 +-
.../IRCE/non-loop-invariant-rhs-instr.ll | 2 +-
llvm/test/Transforms/IRCE/pre_post_loops.ll | 4 ++--
.../IRCE/range_intersect_miscompile.ll | 6 ++---
.../IRCE/ranges_of_different_types.ll | 24 +++++++++----------
.../test/Transforms/IRCE/rc-negative-bound.ll | 8 +++----
.../Transforms/IRCE/stride_more_than_1.ll | 22 ++++++++---------
llvm/test/Transforms/IRCE/unhandled.ll | 2 +-
.../IRCE/unsigned_comparisons_ugt.ll | 10 ++++----
.../IRCE/unsigned_comparisons_ult.ll | 14 +++++------
llvm/test/Transforms/IRCE/wide_indvar.ll | 20 ++++++++--------
16 files changed, 74 insertions(+), 74 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index f907dcc023f295a..73bf097b9c412d3 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -129,7 +129,7 @@ static cl::opt<bool>
PrintScaledBoundaryRangeChecks("irce-print-scaled-boundary-range-checks",
cl::Hidden, cl::init(false));
-static const char *ClonedLoopTag = "irce.loop.clone";
+static const char *ClonedLoopTag = "loop_constrainer.loop.clone";
#define DEBUG_TYPE "irce"
diff --git a/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll b/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll
index 4d0b25b7845edb9..eb72907730c6d5d 100644
--- a/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll
+++ b/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll
@@ -9,7 +9,7 @@
define void @inner_loop(ptr %arr, ptr %a_len_ptr, i32 %n) #0 {
; CHECK-LABEL: inner_loop(
; CHECK-LABEL: in.bounds.postloop
-; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit.loopexit, !llvm.loop !2, !irce.loop.clone !7
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit.loopexit, !llvm.loop !2, !loop_constrainer.loop.clone !7
entry:
%len = load i32, ptr %a_len_ptr, !range !0
@@ -39,9 +39,9 @@ exit: ; preds = %in.bounds, %entry
define void @single_access_with_preloop(ptr %arr, ptr %a_len_ptr, i32 %n, i32 %offset) {
; CHECK-LABEL: @single_access_with_preloop(
; CHECK-LABEL: in.bounds.preloop
-; CHECK: br i1 [[COND:%[^ ]+]], label %loop.preloop, label %preloop.exit.selector, !llvm.loop !8, !irce.loop.clone !7
+; CHECK: br i1 [[COND:%[^ ]+]], label %loop.preloop, label %preloop.exit.selector, !llvm.loop !8, !loop_constrainer.loop.clone !7
; CHECK-LABEL: in.bounds.postloop
-; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit.loopexit, !llvm.loop !9, !irce.loop.clone !7
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit.loopexit, !llvm.loop !9, !loop_constrainer.loop.clone !7
entry:
%len = load i32, ptr %a_len_ptr, !range !0
%first.itr.check = icmp sgt i32 %n, 0
diff --git a/llvm/test/Transforms/IRCE/conjunctive-checks.ll b/llvm/test/Transforms/IRCE/conjunctive-checks.ll
index d914ed87815bfeb..a384c99eb0d560b 100644
--- a/llvm/test/Transforms/IRCE/conjunctive-checks.ll
+++ b/llvm/test/Transforms/IRCE/conjunctive-checks.ll
@@ -64,7 +64,7 @@ define void @f_0(ptr %arr, ptr %a_len_ptr, i32 %n, ptr %cond_buf) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_FOR_ABC_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]], !irce.loop.clone [[META7:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]], !loop_constrainer.loop.clone [[META7:![0-9]+]]
;
entry:
%len = load i32, ptr %a_len_ptr, !range !0
@@ -158,7 +158,7 @@ define void @f_1(
; CHECK-NEXT: [[ADDR_B_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR_B]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 -1, ptr [[ADDR_B_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone [[META7]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META7]]
;
ptr %arr_a, ptr %a_len_ptr, ptr %arr_b, ptr %b_len_ptr, i32 %n) {
diff --git a/llvm/test/Transforms/IRCE/correct-loop-info.ll b/llvm/test/Transforms/IRCE/correct-loop-info.ll
index 515ad15473cb813..c75de167681ad60 100644
--- a/llvm/test/Transforms/IRCE/correct-loop-info.ll
+++ b/llvm/test/Transforms/IRCE/correct-loop-info.ll
@@ -99,7 +99,7 @@ define void @baz() personality ptr @ham {
; CHECK: bb8.preloop:
; CHECK-NEXT: [[TMP9_PRELOOP:%.*]] = icmp slt i32 [[TMP6_PRELOOP]], 84
; CHECK-NEXT: [[TMP4:%.*]] = icmp slt i32 [[TMP6_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP4]], label [[INNERHEADER_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP0:![0-9]+]], !irce.loop.clone [[META5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[INNERHEADER_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone [[META5:![0-9]+]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[TMP6_PRELOOP_LCSSA:%.*]] = phi i32 [ [[TMP6_PRELOOP]], [[BB8_PRELOOP]] ]
; CHECK-NEXT: [[TMP5:%.*]] = icmp slt i32 [[TMP6_PRELOOP_LCSSA]], 84
@@ -120,7 +120,7 @@ define void @baz() personality ptr @ham {
; CHECK-NEXT: br i1 [[TMP7_POSTLOOP]], label [[BB8_POSTLOOP]], label [[EXIT3_LOOPEXIT4:%.*]]
; CHECK: bb8.postloop:
; CHECK-NEXT: [[TMP9_POSTLOOP:%.*]] = icmp slt i32 [[TMP6_POSTLOOP]], 84
-; CHECK-NEXT: br i1 [[TMP9_POSTLOOP]], label [[INNERHEADER_POSTLOOP]], label [[BB13_LOOPEXIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]], !irce.loop.clone [[META5]]
+; CHECK-NEXT: br i1 [[TMP9_POSTLOOP]], label [[INNERHEADER_POSTLOOP]], label [[BB13_LOOPEXIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
;
bb:
br label %outerheader
diff --git a/llvm/test/Transforms/IRCE/iv-plus-offset-range-check.ll b/llvm/test/Transforms/IRCE/iv-plus-offset-range-check.ll
index bda7cf82eff97a8..ff8f705d3309922 100644
--- a/llvm/test/Transforms/IRCE/iv-plus-offset-range-check.ll
+++ b/llvm/test/Transforms/IRCE/iv-plus-offset-range-check.ll
@@ -97,7 +97,7 @@ define i8 @test1(i8 %limit, i8 %n) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP0:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%precheck = icmp sgt i8 %limit, 0
@@ -191,7 +191,7 @@ define i8 @test1a(i8 %limit, ptr %p) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%n = load i8, ptr %p, !range !0
@@ -445,7 +445,7 @@ define i8 @test3a(i8 %limit, ptr %p) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%n = load i8, ptr %p, !range !0
@@ -549,7 +549,7 @@ define i8 @test4(i8 %limit, i8 %n) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%precheck = icmp sgt i8 %limit, 0
@@ -648,7 +648,7 @@ define i8 @test4a(i8 %limit, ptr %p) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP10:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%n = load i8, ptr %p, !range !0
@@ -853,7 +853,7 @@ define i8 @test6(i8 %limit, i8 %n) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP11:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%precheck = icmp sgt i8 %limit, 0
@@ -944,7 +944,7 @@ define i8 @test6a(i8 %limit, ptr %p) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%n = load i8, ptr %p, !range !0
@@ -1038,7 +1038,7 @@ define i8 @test_overflow_check_compile_time(i8 %limit, ptr %p) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP14:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP14:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%n = load i8, ptr %p, !range !1
@@ -1147,7 +1147,7 @@ define i8 @test_overflow_check_runtime(i8 %limit, ptr %p) {
; CHECK: inbounds.postloop:
; CHECK-NEXT: [[IDX_NEXT_POSTLOOP]] = add nuw i8 [[IDX_POSTLOOP]], 1
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i8 [[IDX_NEXT_POSTLOOP]], [[LIMIT]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP15:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOP15:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
%n = load i8, ptr %p, !range !1
diff --git a/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll b/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll
index 94dc9bb44e1f512..0570bc65bcdf3a4 100644
--- a/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll
+++ b/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll
@@ -69,7 +69,7 @@ define void @multiple_access_no_preloop(
; CHECK-NEXT: [[ADDR_B_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR_B]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 -1, ptr [[ADDR_B_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]], !irce.loop.clone [[META7:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]], !loop_constrainer.loop.clone [[META7:![0-9]+]]
;
ptr %arr_a, ptr %a_len_ptr, ptr %arr_b, ptr %b_len_ptr, i32 %n) {
diff --git a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll
index d9d2a27e973b632..c28fc59014f5c6f 100644
--- a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll
+++ b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll
@@ -80,7 +80,7 @@ define i32 @test_01(i32 %A, i64 %Len, ptr %array) {
; CHECK-NEXT: [[RES2_POSTLOOP]] = mul i32 [[RES_POSTLOOP]], 3
; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[A]] to i64
; CHECK-NEXT: [[CMP2_POSTLOOP:%.*]] = icmp ugt i64 [[INDVAR_NEXT_POSTLOOP]], [[TMP10]]
-; CHECK-NEXT: br i1 [[CMP2_POSTLOOP]], label [[LOOPEXIT_LOOPEXIT]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP0:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[CMP2_POSTLOOP]], label [[LOOPEXIT_LOOPEXIT]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone !5
;
preheader:
%tripcheck = icmp sgt i64 %Len, 2
diff --git a/llvm/test/Transforms/IRCE/pre_post_loops.ll b/llvm/test/Transforms/IRCE/pre_post_loops.ll
index f6368cf28776b6e..da2faa34719afec 100644
--- a/llvm/test/Transforms/IRCE/pre_post_loops.ll
+++ b/llvm/test/Transforms/IRCE/pre_post_loops.ll
@@ -55,7 +55,7 @@ define void @test_01(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 2147483647
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !irce.loop.clone [[META6:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone [[META6:![0-9]+]]
;
entry:
@@ -122,7 +122,7 @@ define void @test_02(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp sgt i32 [[IDX_NEXT_PRELOOP]], -1
; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i32 [[IDX_NEXT_PRELOOP]], -1
-; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[IDX_NEXT_PRELOOP_LCSSA]], -1
diff --git a/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll b/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll
index 75c7be87e2c030b..68613feacba3284 100644
--- a/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll
+++ b/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll
@@ -83,7 +83,7 @@ define void @test_01() {
; CHECK: loop_latch.postloop:
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i32 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[LOOP_COND_POSTLOOP:%.*]] = icmp ult i32 [[IV_NEXT_POSTLOOP]], 400
-; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP0:![0-9]+]], !irce.loop.clone [[META5:![0-9]+]]
+; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone [[META5:![0-9]+]]
;
entry:
@@ -333,7 +333,7 @@ define void @test_04(ptr %p) {
; CHECK: loop_latch.postloop:
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i32 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[LOOP_COND_POSTLOOP:%.*]] = icmp ult i32 [[IV_NEXT_POSTLOOP]], 400
-; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP6:![0-9]+]], !irce.loop.clone [[META5]]
+; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP6:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
;
entry:
@@ -446,7 +446,7 @@ define void @test_05(ptr %p) {
; CHECK: loop_latch.postloop:
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i32 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[LOOP_COND_POSTLOOP:%.*]] = icmp ult i32 [[IV_NEXT_POSTLOOP]], 400
-; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone [[META5]]
+; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META5]]
;
entry:
diff --git a/llvm/test/Transforms/IRCE/ranges_of_different_types.ll b/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
index b66aa94aaf0f864..c3246969155f76f 100644
--- a/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
+++ b/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
@@ -71,7 +71,7 @@ define void @test_01(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !irce.loop.clone [[META6:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone [[META6:![0-9]+]]
;
entry:
@@ -173,7 +173,7 @@ define void @test_02(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp slt i32 [[IDX_NEXT_PRELOOP]], 101
; CHECK-NEXT: [[TMP5:%.*]] = icmp slt i32 [[IDX_NEXT_PRELOOP]], 13
-; CHECK-NEXT: br i1 [[TMP5]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 101
@@ -194,7 +194,7 @@ define void @test_02(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -290,7 +290,7 @@ define void @test_03(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -390,7 +390,7 @@ define void @test_04(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP]], 101
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP]], 13
-; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP10:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP_LCSSA]], 101
@@ -411,7 +411,7 @@ define void @test_04(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -492,7 +492,7 @@ define void @test_05(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -579,7 +579,7 @@ define void @test_06(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp slt i32 [[IDX_NEXT_PRELOOP]], 101
; CHECK-NEXT: [[TMP5:%.*]] = icmp slt i32 [[IDX_NEXT_PRELOOP]], 13
-; CHECK-NEXT: br i1 [[TMP5]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP13:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP13:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 101
@@ -600,7 +600,7 @@ define void @test_06(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP14:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP14:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -681,7 +681,7 @@ define void @test_07(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP15:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP15:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -766,7 +766,7 @@ define void @test_08(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP]], 101
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP]], 13
-; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP16:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP16:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP_LCSSA]], 101
@@ -787,7 +787,7 @@ define void @test_08(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], 101
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP17:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP17:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
diff --git a/llvm/test/Transforms/IRCE/rc-negative-bound.ll b/llvm/test/Transforms/IRCE/rc-negative-bound.ll
index 0c94d2b4f8c7a2f..b3b2a18dad87b32 100644
--- a/llvm/test/Transforms/IRCE/rc-negative-bound.ll
+++ b/llvm/test/Transforms/IRCE/rc-negative-bound.ll
@@ -167,7 +167,7 @@ define void @test_03(ptr %arr, i32 %n, i32 %bound) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !irce.loop.clone [[META6:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone [[META6:![0-9]+]]
;
entry:
@@ -253,7 +253,7 @@ define void @test_04(ptr %arr, i32 %n, i32 %bound) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -447,7 +447,7 @@ define void @test_07(ptr %arr, i32 %n, i32 %bound) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
%first.itr.check = icmp sgt i32 %n, 0
@@ -535,7 +535,7 @@ define void @test_08(ptr %arr, i32 %n, i32 %bound) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
%first.itr.check = icmp sgt i32 %n, 0
diff --git a/llvm/test/Transforms/IRCE/stride_more_than_1.ll b/llvm/test/Transforms/IRCE/stride_more_than_1.ll
index da6b5603e92600b..92cd410b12f66fe 100644
--- a/llvm/test/Transforms/IRCE/stride_more_than_1.ll
+++ b/llvm/test/Transforms/IRCE/stride_more_than_1.ll
@@ -61,7 +61,7 @@ define void @test_01(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 100
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
@@ -137,7 +137,7 @@ define void @test_02(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 2147483640
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
@@ -214,7 +214,7 @@ define void @test_03(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 2147483647
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
@@ -292,7 +292,7 @@ define void @test_04(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], 2147483647
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone !6
;
@@ -363,7 +363,7 @@ define void @test_05(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp sgt i32 [[IDX_NEXT_PRELOOP]], -1
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone !6
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt i32 [[IDX_NEXT_PRELOOP_LCSSA]], -1
@@ -440,7 +440,7 @@ define void @test_06(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], 6
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP14:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP14:![0-9]+]], !loop_constrainer.loop.clone !6
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 6
@@ -564,7 +564,7 @@ define void @test_08(ptr %arr, ptr %a_len_ptr) {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], 6
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP15:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP15:![0-9]+]], !loop_constrainer.loop.clone !6
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 6
@@ -673,7 +673,7 @@ define i32 @test_09(ptr %p, ptr %capacity_p, ptr %num_elements_p) {
; CHECK-NEXT: store i32 1, ptr [[EL_PTR_POSTLOOP]], align 4
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add nuw nsw i32 [[IV_POSTLOOP]], 4
; CHECK-NEXT: [[LOOP_COND_POSTLOOP:%.*]] = icmp slt i32 [[IV_NEXT_POSTLOOP]], [[NUM_ELEMENTS]]
-; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
%capacity = load i32, ptr %capacity_p, !range !4
@@ -769,7 +769,7 @@ define i32 @test_10(ptr %p, ptr %capacity_p, ptr %num_elements_p) {
; CHECK-NEXT: store i32 1, ptr [[EL_PTR_POSTLOOP]], align 4
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add nuw nsw i32 [[IV_POSTLOOP]], 4
; CHECK-NEXT: [[LOOP_COND_POSTLOOP:%.*]] = icmp slt i32 [[IV_NEXT_POSTLOOP]], [[NUM_ELEMENTS]]
-; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP19:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP19:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
%capacity = load i32, ptr %capacity_p, !range !4
@@ -866,7 +866,7 @@ define i32 @test_11(ptr %p, ptr %capacity_p, ptr %num_elements_p) {
; CHECK-NEXT: store i32 1, ptr [[EL_PTR_POSTLOOP]], align 4
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add nuw nsw i32 [[IV_POSTLOOP]], 4
; CHECK-NEXT: [[LOOP_COND_POSTLOOP:%.*]] = icmp slt i32 [[IV_NEXT_POSTLOOP]], [[NUM_ELEMENTS]]
-; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP20:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[LOOP_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP20:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
%capacity = load i32, ptr %capacity_p, !range !4
@@ -953,7 +953,7 @@ define i32 @binop_or_is_iv_base(ptr %p, i32 %end) {
; CHECK-NEXT: [[IV_ADD_POSTLOOP]] = add i32 [[IV_POSTLOOP]], 8
; CHECK-NEXT: [[IV_OR_POSTLOOP:%.*]] = or i32 [[IV_ADD_POSTLOOP]], 7
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp slt i32 [[IV_OR_POSTLOOP]], [[END]]
-; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[COMMON_RET_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP21:![0-9]+]], !irce.loop.clone !6
+; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LOOP_HEADER_POSTLOOP]], label [[COMMON_RET_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP21:![0-9]+]], !loop_constrainer.loop.clone !6
;
entry:
%n = load atomic i32, ptr %p unordered, align 8, !range !1
diff --git a/llvm/test/Transforms/IRCE/unhandled.ll b/llvm/test/Transforms/IRCE/unhandled.ll
index a604e303d1b38c2..0b06ce9ac253dc1 100644
--- a/llvm/test/Transforms/IRCE/unhandled.ll
+++ b/llvm/test/Transforms/IRCE/unhandled.ll
@@ -87,7 +87,7 @@ define void @already_cloned(ptr %arr, ptr %a_len_ptr, i32 %n) {
%addr = getelementptr i32, ptr %arr, i32 %idx
store i32 0, ptr %addr
%next = icmp slt i32 %idx.next, %n
- br i1 %next, label %loop, label %exit, !irce.loop.clone !{}
+ br i1 %next, label %loop, label %exit, !loop_constrainer.loop.clone !{}
out.of.bounds:
ret void
diff --git a/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll b/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll
index ce7b40ef1fa88a8..7236cf4817adee3 100644
--- a/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll
+++ b/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll
@@ -60,7 +60,7 @@ define void @test_01(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_POSTLOOP]], 100
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP1:![0-9]+]], !irce.loop.clone [[META6:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone [[META6:![0-9]+]]
;
entry:
@@ -130,7 +130,7 @@ define void @test_02(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], 0
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 0
@@ -215,7 +215,7 @@ define void @test_03(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_POSTLOOP]], -2147483648
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -285,7 +285,7 @@ define void @test_04(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], 0
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 0
@@ -406,7 +406,7 @@ define void @test_06(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: store i32 0, ptr [[ADDR_PRELOOP]], align 4
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], 0
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], 0
-; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP10:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 0
diff --git a/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll b/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll
index 7920ac362f453f6..546e1b0faa1c344 100644
--- a/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll
+++ b/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll
@@ -62,7 +62,7 @@ define void @test_01(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], 100
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !irce.loop.clone [[META6:![0-9]+]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP1:![0-9]+]], !loop_constrainer.loop.clone [[META6:![0-9]+]]
;
entry:
@@ -133,7 +133,7 @@ define void @test_02(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[TMP1]], true
-; CHECK-NEXT: br i1 [[TMP2]], label [[PRELOOP_EXIT_SELECTOR:%.*]], label [[LOOP_PRELOOP]], !llvm.loop [[LOOP7:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[PRELOOP_EXIT_SELECTOR:%.*]], label [[LOOP_PRELOOP]], !llvm.loop [[LOOP7:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 0
@@ -217,7 +217,7 @@ define void @test_03(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], 2147483647
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -293,7 +293,7 @@ define void @test_04(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], -2147483648
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -364,7 +364,7 @@ define void @test_05(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[NEXT_PRELOOP:%.*]] = icmp ult i32 [[IDX_NEXT_PRELOOP]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP]], [[EXIT_PRELOOP_AT]]
; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[TMP1]], true
-; CHECK-NEXT: br i1 [[TMP2]], label [[PRELOOP_EXIT_SELECTOR:%.*]], label [[LOOP_PRELOOP]], !llvm.loop [[LOOP10:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[PRELOOP_EXIT_SELECTOR:%.*]], label [[LOOP_PRELOOP]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IDX_NEXT_PRELOOP_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT_PRELOOP]], [[IN_BOUNDS_PRELOOP]] ]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[IDX_NEXT_PRELOOP_LCSSA]], 0
@@ -448,7 +448,7 @@ define void @test_06(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], -1
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
@@ -573,7 +573,7 @@ define void @test_08(ptr %arr, ptr %a_len_ptr) #0 {
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr i32, ptr [[ARR]], i32 [[IDX_POSTLOOP]]
; CHECK-NEXT: store i32 0, ptr [[ADDR_POSTLOOP]], align 4
; CHECK-NEXT: [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], -100
-; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !irce.loop.clone [[META6]]
+; CHECK-NEXT: br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone [[META6]]
;
entry:
diff --git a/llvm/test/Transforms/IRCE/wide_indvar.ll b/llvm/test/Transforms/IRCE/wide_indvar.ll
index 415ed5750dcc496..a0200b2b7b0d846 100644
--- a/llvm/test/Transforms/IRCE/wide_indvar.ll
+++ b/llvm/test/Transforms/IRCE/wide_indvar.ll
@@ -94,7 +94,7 @@ define i32 @test_increasing_slt_slt_wide_simple_postloop() {
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp slt i32 [[NARROW_IV_POSTLOOP]], 100
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP0:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -175,7 +175,7 @@ define i32 @test_increasing_slt_slt_wide_non-negative(ptr %n_ptr, ptr %m_ptr) {
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp slt i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP8:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -268,7 +268,7 @@ define i32 @test_increasing_slt_slt_wide_general(ptr %n_ptr, ptr %m_ptr) {
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp slt i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP9:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -367,7 +367,7 @@ define i32 @test_increasing_slt_slt_wide_general_preloop(ptr %n_ptr, ptr %m_ptr)
; CHECK-NEXT: [[LATCH_COND_PRELOOP:%.*]] = icmp slt i32 [[NARROW_IV_PRELOOP]], [[N]]
; CHECK-NEXT: [[WIDE_NARROW_IV_PRELOOP:%.*]] = sext i32 [[NARROW_IV_PRELOOP]] to i64
; CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[WIDE_NARROW_IV_PRELOOP]], [[EXIT_PRELOOP_AT]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR]], !llvm.loop [[LOOP10:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[TMP9]], label [[LOOP_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR]], !llvm.loop [[LOOP10:![0-9]+]], !loop_constrainer.loop.clone !5
; CHECK: preloop.exit.selector:
; CHECK-NEXT: [[IV_NEXT_PRELOOP_LCSSA:%.*]] = phi i64 [ [[IV_NEXT_PRELOOP]], [[BACKEDGE_PRELOOP]] ]
; CHECK-NEXT: [[NARROW_IV_PRELOOP_LCSSA]] = phi i32 [ [[NARROW_IV_PRELOOP]], [[BACKEDGE_PRELOOP]] ]
@@ -389,7 +389,7 @@ define i32 @test_increasing_slt_slt_wide_general_preloop(ptr %n_ptr, ptr %m_ptr)
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp slt i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP11:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP11:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -519,7 +519,7 @@ define i32 @test_increasing_slt_slt_wide_multiple_checks(ptr %n_ptr, ptr %m1_ptr
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp slt i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -688,7 +688,7 @@ define i32 @test_increasing_ult_ult_wide_simple_postloop() {
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp ult i32 [[NARROW_IV_POSTLOOP]], 100
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP13:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP13:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -769,7 +769,7 @@ define i32 @test_increasing_ult_ult_wide_non-negative(ptr %n_ptr, ptr %m_ptr) {
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp ult i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP14:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP14:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -859,7 +859,7 @@ define i32 @test_increasing_ult_ult_wide_general(ptr %n_ptr, ptr %m_ptr) {
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp ult i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP15:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP15:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
@@ -980,7 +980,7 @@ define i32 @test_increasing_ult_ult_wide_multiple_checks(ptr %n_ptr, ptr %m1_ptr
; CHECK-NEXT: [[IV_NEXT_POSTLOOP]] = add i64 [[IV_POSTLOOP]], 1
; CHECK-NEXT: [[NARROW_IV_POSTLOOP]] = trunc i64 [[IV_NEXT_POSTLOOP]] to i32
; CHECK-NEXT: [[LATCH_COND_POSTLOOP:%.*]] = icmp ult i32 [[NARROW_IV_POSTLOOP]], [[N]]
-; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]], !irce.loop.clone !5
+; CHECK-NEXT: br i1 [[LATCH_COND_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]], !loop_constrainer.loop.clone !5
;
entry:
>From 2d2635de27185d7420dd89af74e4df8c76490add Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 09:30:26 -0700
Subject: [PATCH 7/8] [NFC] Extract LoopStructure from IRCE
Preparatory patch for extracting LoopConstrainer to a separate file to
reuse it outside IRCE.
---
.../llvm/Transforms/Utils/LoopConstrainer.h | 83 +++
.../Scalar/InductiveRangeCheckElimination.cpp | 475 +-----------------
llvm/lib/Transforms/Utils/CMakeLists.txt | 1 +
llvm/lib/Transforms/Utils/LoopConstrainer.cpp | 422 ++++++++++++++++
.../llvm/lib/Transforms/Utils/BUILD.gn | 1 +
5 files changed, 508 insertions(+), 474 deletions(-)
create mode 100644 llvm/include/llvm/Transforms/Utils/LoopConstrainer.h
create mode 100644 llvm/lib/Transforms/Utils/LoopConstrainer.cpp
diff --git a/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h b/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h
new file mode 100644
index 000000000000000..9c58c94edd29143
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h
@@ -0,0 +1,83 @@
+//===- LoopConstrainer.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_LOOP_CONSTRAINER_H
+#define LLVM_TRANSFORMS_UTILS_LOOP_CONSTRAINER_H
+
+#include "llvm/Support/Casting.h"
+#include <optional>
+
+namespace llvm {
+
+class BasicBlock;
+class BranchInst;
+class IntegerType;
+class Loop;
+class ScalarEvolution;
+class Value;
+
+// Keeps track of the structure of a loop. This is similar to llvm::Loop,
+// except that it is more lightweight and can track the state of a loop through
+// changing and potentially invalid IR. This structure also formalizes the
+// kinds of loops we can deal with -- ones that have a single latch that is also
+// an exiting block *and* have a canonical induction variable.
+struct LoopStructure {
+ const char *Tag = "";
+
+ BasicBlock *Header = nullptr;
+ BasicBlock *Latch = nullptr;
+
+ // `Latch's terminator instruction is `LatchBr', and it's `LatchBrExitIdx'th
+ // successor is `LatchExit', the exit block of the loop.
+ BranchInst *LatchBr = nullptr;
+ BasicBlock *LatchExit = nullptr;
+ unsigned LatchBrExitIdx = std::numeric_limits<unsigned>::max();
+
+ // The loop represented by this instance of LoopStructure is semantically
+ // equivalent to:
+ //
+ // intN_ty inc = IndVarIncreasing ? 1 : -1;
+ // pred_ty predicate = IndVarIncreasing ? ICMP_SLT : ICMP_SGT;
+ //
+ // for (intN_ty iv = IndVarStart; predicate(iv, LoopExitAt); iv = IndVarBase)
+ // ... body ...
+
+ Value *IndVarBase = nullptr;
+ Value *IndVarStart = nullptr;
+ Value *IndVarStep = nullptr;
+ Value *LoopExitAt = nullptr;
+ bool IndVarIncreasing = false;
+ bool IsSignedPredicate = true;
+ IntegerType *ExitCountTy = nullptr;
+
+ LoopStructure() = default;
+
+ template <typename M> LoopStructure map(M Map) const {
+ LoopStructure Result;
+ Result.Tag = Tag;
+ Result.Header = cast<BasicBlock>(Map(Header));
+ Result.Latch = cast<BasicBlock>(Map(Latch));
+ Result.LatchBr = cast<BranchInst>(Map(LatchBr));
+ Result.LatchExit = cast<BasicBlock>(Map(LatchExit));
+ Result.LatchBrExitIdx = LatchBrExitIdx;
+ Result.IndVarBase = Map(IndVarBase);
+ Result.IndVarStart = Map(IndVarStart);
+ Result.IndVarStep = Map(IndVarStep);
+ Result.LoopExitAt = Map(LoopExitAt);
+ Result.IndVarIncreasing = IndVarIncreasing;
+ Result.IsSignedPredicate = IsSignedPredicate;
+ Result.ExitCountTy = ExitCountTy;
+ return Result;
+ }
+
+ static std::optional<LoopStructure>
+ parseLoopStructure(ScalarEvolution &, Loop &, bool, const char *&);
+};
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_LOOP_CONSTRAINER_H
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 73bf097b9c412d3..c17feb0d9d03283 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -81,6 +81,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopConstrainer.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
@@ -241,8 +242,6 @@ class InductiveRangeCheck {
SmallVectorImpl<InductiveRangeCheck> &Checks, bool &Changed);
};
-struct LoopStructure;
-
class InductiveRangeCheckElimination {
ScalarEvolution &SE;
BranchProbabilityInfo *BPI;
@@ -584,64 +583,6 @@ static void DisableAllLoopOptsOnLoop(Loop &L) {
namespace {
-// Keeps track of the structure of a loop. This is similar to llvm::Loop,
-// except that it is more lightweight and can track the state of a loop through
-// changing and potentially invalid IR. This structure also formalizes the
-// kinds of loops we can deal with -- ones that have a single latch that is also
-// an exiting block *and* have a canonical induction variable.
-struct LoopStructure {
- const char *Tag = "";
-
- BasicBlock *Header = nullptr;
- BasicBlock *Latch = nullptr;
-
- // `Latch's terminator instruction is `LatchBr', and it's `LatchBrExitIdx'th
- // successor is `LatchExit', the exit block of the loop.
- BranchInst *LatchBr = nullptr;
- BasicBlock *LatchExit = nullptr;
- unsigned LatchBrExitIdx = std::numeric_limits<unsigned>::max();
-
- // The loop represented by this instance of LoopStructure is semantically
- // equivalent to:
- //
- // intN_ty inc = IndVarIncreasing ? 1 : -1;
- // pred_ty predicate = IndVarIncreasing ? ICMP_SLT : ICMP_SGT;
- //
- // for (intN_ty iv = IndVarStart; predicate(iv, LoopExitAt); iv = IndVarBase)
- // ... body ...
-
- Value *IndVarBase = nullptr;
- Value *IndVarStart = nullptr;
- Value *IndVarStep = nullptr;
- Value *LoopExitAt = nullptr;
- bool IndVarIncreasing = false;
- bool IsSignedPredicate = true;
- IntegerType *ExitCountTy = nullptr;
-
- LoopStructure() = default;
-
- template <typename M> LoopStructure map(M Map) const {
- LoopStructure Result;
- Result.Tag = Tag;
- Result.Header = cast<BasicBlock>(Map(Header));
- Result.Latch = cast<BasicBlock>(Map(Latch));
- Result.LatchBr = cast<BranchInst>(Map(LatchBr));
- Result.LatchExit = cast<BasicBlock>(Map(LatchExit));
- Result.LatchBrExitIdx = LatchBrExitIdx;
- Result.IndVarBase = Map(IndVarBase);
- Result.IndVarStart = Map(IndVarStart);
- Result.IndVarStep = Map(IndVarStep);
- Result.LoopExitAt = Map(LoopExitAt);
- Result.IndVarIncreasing = IndVarIncreasing;
- Result.IsSignedPredicate = IsSignedPredicate;
- Result.ExitCountTy = ExitCountTy;
- return Result;
- }
-
- static std::optional<LoopStructure>
- parseLoopStructure(ScalarEvolution &, Loop &, bool, const char *&);
-};
-
/// This class is used to constrain loops to run within a given iteration space.
/// The algorithm this class implements is given a Loop and a range [Begin,
/// End). The algorithm then tries to break out a "main loop" out of the loop
@@ -785,420 +726,6 @@ class LoopConstrainer {
} // end anonymous namespace
-/// Given a loop with an deccreasing induction variable, is it possible to
-/// safely calculate the bounds of a new loop using the given Predicate.
-static bool isSafeDecreasingBound(const SCEV *Start,
- const SCEV *BoundSCEV, const SCEV *Step,
- ICmpInst::Predicate Pred,
- unsigned LatchBrExitIdx,
- Loop *L, ScalarEvolution &SE) {
- if (Pred != ICmpInst::ICMP_SLT && Pred != ICmpInst::ICMP_SGT &&
- Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_UGT)
- return false;
-
- if (!SE.isAvailableAtLoopEntry(BoundSCEV, L))
- return false;
-
- assert(SE.isKnownNegative(Step) && "expecting negative step");
-
- LLVM_DEBUG(dbgs() << "isSafeDecreasingBound with:\n");
- LLVM_DEBUG(dbgs() << "Start: " << *Start << "\n");
- LLVM_DEBUG(dbgs() << "Step: " << *Step << "\n");
- LLVM_DEBUG(dbgs() << "BoundSCEV: " << *BoundSCEV << "\n");
- LLVM_DEBUG(dbgs() << "Pred: " << Pred << "\n");
- LLVM_DEBUG(dbgs() << "LatchExitBrIdx: " << LatchBrExitIdx << "\n");
-
- bool IsSigned = ICmpInst::isSigned(Pred);
- // The predicate that we need to check that the induction variable lies
- // within bounds.
- ICmpInst::Predicate BoundPred =
- IsSigned ? CmpInst::ICMP_SGT : CmpInst::ICMP_UGT;
-
- if (LatchBrExitIdx == 1)
- return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, BoundSCEV);
-
- assert(LatchBrExitIdx == 0 &&
- "LatchBrExitIdx should be either 0 or 1");
-
- const SCEV *StepPlusOne = SE.getAddExpr(Step, SE.getOne(Step->getType()));
- unsigned BitWidth = cast<IntegerType>(BoundSCEV->getType())->getBitWidth();
- APInt Min = IsSigned ? APInt::getSignedMinValue(BitWidth) :
- APInt::getMinValue(BitWidth);
- const SCEV *Limit = SE.getMinusSCEV(SE.getConstant(Min), StepPlusOne);
-
- const SCEV *MinusOne =
- SE.getMinusSCEV(BoundSCEV, SE.getOne(BoundSCEV->getType()));
-
- return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, MinusOne) &&
- SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit);
-
-}
-
-/// Given a loop with an increasing induction variable, is it possible to
-/// safely calculate the bounds of a new loop using the given Predicate.
-static bool isSafeIncreasingBound(const SCEV *Start,
- const SCEV *BoundSCEV, const SCEV *Step,
- ICmpInst::Predicate Pred,
- unsigned LatchBrExitIdx,
- Loop *L, ScalarEvolution &SE) {
- if (Pred != ICmpInst::ICMP_SLT && Pred != ICmpInst::ICMP_SGT &&
- Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_UGT)
- return false;
-
- if (!SE.isAvailableAtLoopEntry(BoundSCEV, L))
- return false;
-
- LLVM_DEBUG(dbgs() << "isSafeIncreasingBound with:\n");
- LLVM_DEBUG(dbgs() << "Start: " << *Start << "\n");
- LLVM_DEBUG(dbgs() << "Step: " << *Step << "\n");
- LLVM_DEBUG(dbgs() << "BoundSCEV: " << *BoundSCEV << "\n");
- LLVM_DEBUG(dbgs() << "Pred: " << Pred << "\n");
- LLVM_DEBUG(dbgs() << "LatchExitBrIdx: " << LatchBrExitIdx << "\n");
-
- bool IsSigned = ICmpInst::isSigned(Pred);
- // The predicate that we need to check that the induction variable lies
- // within bounds.
- ICmpInst::Predicate BoundPred =
- IsSigned ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT;
-
- if (LatchBrExitIdx == 1)
- return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, BoundSCEV);
-
- assert(LatchBrExitIdx == 0 && "LatchBrExitIdx should be 0 or 1");
-
- const SCEV *StepMinusOne =
- SE.getMinusSCEV(Step, SE.getOne(Step->getType()));
- unsigned BitWidth = cast<IntegerType>(BoundSCEV->getType())->getBitWidth();
- APInt Max = IsSigned ? APInt::getSignedMaxValue(BitWidth) :
- APInt::getMaxValue(BitWidth);
- const SCEV *Limit = SE.getMinusSCEV(SE.getConstant(Max), StepMinusOne);
-
- return (SE.isLoopEntryGuardedByCond(L, BoundPred, Start,
- SE.getAddExpr(BoundSCEV, Step)) &&
- SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit));
-}
-
-/// Returns estimate for max latch taken count of the loop of the narrowest
-/// available type. If the latch block has such estimate, it is returned.
-/// Otherwise, we use max exit count of whole loop (that is potentially of wider
-/// type than latch check itself), which is still better than no estimate.
-static const SCEV *getNarrowestLatchMaxTakenCountEstimate(ScalarEvolution &SE,
- const Loop &L) {
- const SCEV *FromBlock =
- SE.getExitCount(&L, L.getLoopLatch(), ScalarEvolution::SymbolicMaximum);
- if (isa<SCEVCouldNotCompute>(FromBlock))
- return SE.getSymbolicMaxBackedgeTakenCount(&L);
- return FromBlock;
-}
-
-std::optional<LoopStructure>
-LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
- bool AllowUnsignedLatchCond,
- const char *&FailureReason) {
- if (!L.isLoopSimplifyForm()) {
- FailureReason = "loop not in LoopSimplify form";
- return std::nullopt;
- }
-
- BasicBlock *Latch = L.getLoopLatch();
- assert(Latch && "Simplified loops only have one latch!");
-
- if (Latch->getTerminator()->getMetadata(ClonedLoopTag)) {
- FailureReason = "loop has already been cloned";
- return std::nullopt;
- }
-
- if (!L.isLoopExiting(Latch)) {
- FailureReason = "no loop latch";
- return std::nullopt;
- }
-
- BasicBlock *Header = L.getHeader();
- BasicBlock *Preheader = L.getLoopPreheader();
- if (!Preheader) {
- FailureReason = "no preheader";
- return std::nullopt;
- }
-
- BranchInst *LatchBr = dyn_cast<BranchInst>(Latch->getTerminator());
- if (!LatchBr || LatchBr->isUnconditional()) {
- FailureReason = "latch terminator not conditional branch";
- return std::nullopt;
- }
-
- unsigned LatchBrExitIdx = LatchBr->getSuccessor(0) == Header ? 1 : 0;
-
- ICmpInst *ICI = dyn_cast<ICmpInst>(LatchBr->getCondition());
- if (!ICI || !isa<IntegerType>(ICI->getOperand(0)->getType())) {
- FailureReason = "latch terminator branch not conditional on integral icmp";
- return std::nullopt;
- }
-
- const SCEV *MaxBETakenCount = getNarrowestLatchMaxTakenCountEstimate(SE, L);
- if (isa<SCEVCouldNotCompute>(MaxBETakenCount)) {
- FailureReason = "could not compute latch count";
- return std::nullopt;
- }
- assert(SE.getLoopDisposition(MaxBETakenCount, &L) ==
- ScalarEvolution::LoopInvariant &&
- "loop variant exit count doesn't make sense!");
-
- ICmpInst::Predicate Pred = ICI->getPredicate();
- Value *LeftValue = ICI->getOperand(0);
- const SCEV *LeftSCEV = SE.getSCEV(LeftValue);
- IntegerType *IndVarTy = cast<IntegerType>(LeftValue->getType());
-
- Value *RightValue = ICI->getOperand(1);
- const SCEV *RightSCEV = SE.getSCEV(RightValue);
-
- // We canonicalize `ICI` such that `LeftSCEV` is an add recurrence.
- if (!isa<SCEVAddRecExpr>(LeftSCEV)) {
- if (isa<SCEVAddRecExpr>(RightSCEV)) {
- std::swap(LeftSCEV, RightSCEV);
- std::swap(LeftValue, RightValue);
- Pred = ICmpInst::getSwappedPredicate(Pred);
- } else {
- FailureReason = "no add recurrences in the icmp";
- return std::nullopt;
- }
- }
-
- auto HasNoSignedWrap = [&](const SCEVAddRecExpr *AR) {
- if (AR->getNoWrapFlags(SCEV::FlagNSW))
- return true;
-
- IntegerType *Ty = cast<IntegerType>(AR->getType());
- IntegerType *WideTy =
- IntegerType::get(Ty->getContext(), Ty->getBitWidth() * 2);
-
- const SCEVAddRecExpr *ExtendAfterOp =
- dyn_cast<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
- if (ExtendAfterOp) {
- const SCEV *ExtendedStart = SE.getSignExtendExpr(AR->getStart(), WideTy);
- const SCEV *ExtendedStep =
- SE.getSignExtendExpr(AR->getStepRecurrence(SE), WideTy);
-
- bool NoSignedWrap = ExtendAfterOp->getStart() == ExtendedStart &&
- ExtendAfterOp->getStepRecurrence(SE) == ExtendedStep;
-
- if (NoSignedWrap)
- return true;
- }
-
- // We may have proved this when computing the sign extension above.
- return AR->getNoWrapFlags(SCEV::FlagNSW) != SCEV::FlagAnyWrap;
- };
-
- // `ICI` is interpreted as taking the backedge if the *next* value of the
- // induction variable satisfies some constraint.
-
- const SCEVAddRecExpr *IndVarBase = cast<SCEVAddRecExpr>(LeftSCEV);
- if (IndVarBase->getLoop() != &L) {
- FailureReason = "LHS in cmp is not an AddRec for this loop";
- return std::nullopt;
- }
- if (!IndVarBase->isAffine()) {
- FailureReason = "LHS in icmp not induction variable";
- return std::nullopt;
- }
- const SCEV* StepRec = IndVarBase->getStepRecurrence(SE);
- if (!isa<SCEVConstant>(StepRec)) {
- FailureReason = "LHS in icmp not induction variable";
- return std::nullopt;
- }
- ConstantInt *StepCI = cast<SCEVConstant>(StepRec)->getValue();
-
- if (ICI->isEquality() && !HasNoSignedWrap(IndVarBase)) {
- FailureReason = "LHS in icmp needs nsw for equality predicates";
- return std::nullopt;
- }
-
- assert(!StepCI->isZero() && "Zero step?");
- bool IsIncreasing = !StepCI->isNegative();
- bool IsSignedPredicate;
- const SCEV *StartNext = IndVarBase->getStart();
- const SCEV *Addend = SE.getNegativeSCEV(IndVarBase->getStepRecurrence(SE));
- const SCEV *IndVarStart = SE.getAddExpr(StartNext, Addend);
- const SCEV *Step = SE.getSCEV(StepCI);
-
- const SCEV *FixedRightSCEV = nullptr;
-
- // If RightValue resides within loop (but still being loop invariant),
- // regenerate it as preheader.
- if (auto *I = dyn_cast<Instruction>(RightValue))
- if (L.contains(I->getParent()))
- FixedRightSCEV = RightSCEV;
-
- if (IsIncreasing) {
- bool DecreasedRightValueByOne = false;
- if (StepCI->isOne()) {
- // Try to turn eq/ne predicates to those we can work with.
- if (Pred == ICmpInst::ICMP_NE && LatchBrExitIdx == 1)
- // while (++i != len) { while (++i < len) {
- // ... ---> ...
- // } }
- // If both parts are known non-negative, it is profitable to use
- // unsigned comparison in increasing loop. This allows us to make the
- // comparison check against "RightSCEV + 1" more optimistic.
- if (isKnownNonNegativeInLoop(IndVarStart, &L, SE) &&
- isKnownNonNegativeInLoop(RightSCEV, &L, SE))
- Pred = ICmpInst::ICMP_ULT;
- else
- Pred = ICmpInst::ICMP_SLT;
- else if (Pred == ICmpInst::ICMP_EQ && LatchBrExitIdx == 0) {
- // while (true) { while (true) {
- // if (++i == len) ---> if (++i > len - 1)
- // break; break;
- // ... ...
- // } }
- if (IndVarBase->getNoWrapFlags(SCEV::FlagNUW) &&
- cannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/false)) {
- Pred = ICmpInst::ICMP_UGT;
- RightSCEV = SE.getMinusSCEV(RightSCEV,
- SE.getOne(RightSCEV->getType()));
- DecreasedRightValueByOne = true;
- } else if (cannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/true)) {
- Pred = ICmpInst::ICMP_SGT;
- RightSCEV = SE.getMinusSCEV(RightSCEV,
- SE.getOne(RightSCEV->getType()));
- DecreasedRightValueByOne = true;
- }
- }
- }
-
- bool LTPred = (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT);
- bool GTPred = (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_UGT);
- bool FoundExpectedPred =
- (LTPred && LatchBrExitIdx == 1) || (GTPred && LatchBrExitIdx == 0);
-
- if (!FoundExpectedPred) {
- FailureReason = "expected icmp slt semantically, found something else";
- return std::nullopt;
- }
-
- IsSignedPredicate = ICmpInst::isSigned(Pred);
- if (!IsSignedPredicate && !AllowUnsignedLatchCond) {
- FailureReason = "unsigned latch conditions are explicitly prohibited";
- return std::nullopt;
- }
-
- if (!isSafeIncreasingBound(IndVarStart, RightSCEV, Step, Pred,
- LatchBrExitIdx, &L, SE)) {
- FailureReason = "Unsafe loop bounds";
- return std::nullopt;
- }
- if (LatchBrExitIdx == 0) {
- // We need to increase the right value unless we have already decreased
- // it virtually when we replaced EQ with SGT.
- if (!DecreasedRightValueByOne)
- FixedRightSCEV =
- SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
- } else {
- assert(!DecreasedRightValueByOne &&
- "Right value can be decreased only for LatchBrExitIdx == 0!");
- }
- } else {
- bool IncreasedRightValueByOne = false;
- if (StepCI->isMinusOne()) {
- // Try to turn eq/ne predicates to those we can work with.
- if (Pred == ICmpInst::ICMP_NE && LatchBrExitIdx == 1)
- // while (--i != len) { while (--i > len) {
- // ... ---> ...
- // } }
- // We intentionally don't turn the predicate into UGT even if we know
- // that both operands are non-negative, because it will only pessimize
- // our check against "RightSCEV - 1".
- Pred = ICmpInst::ICMP_SGT;
- else if (Pred == ICmpInst::ICMP_EQ && LatchBrExitIdx == 0) {
- // while (true) { while (true) {
- // if (--i == len) ---> if (--i < len + 1)
- // break; break;
- // ... ...
- // } }
- if (IndVarBase->getNoWrapFlags(SCEV::FlagNUW) &&
- cannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ false)) {
- Pred = ICmpInst::ICMP_ULT;
- RightSCEV = SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
- IncreasedRightValueByOne = true;
- } else if (cannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ true)) {
- Pred = ICmpInst::ICMP_SLT;
- RightSCEV = SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
- IncreasedRightValueByOne = true;
- }
- }
- }
-
- bool LTPred = (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT);
- bool GTPred = (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_UGT);
-
- bool FoundExpectedPred =
- (GTPred && LatchBrExitIdx == 1) || (LTPred && LatchBrExitIdx == 0);
-
- if (!FoundExpectedPred) {
- FailureReason = "expected icmp sgt semantically, found something else";
- return std::nullopt;
- }
-
- IsSignedPredicate =
- Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT;
-
- if (!IsSignedPredicate && !AllowUnsignedLatchCond) {
- FailureReason = "unsigned latch conditions are explicitly prohibited";
- return std::nullopt;
- }
-
- if (!isSafeDecreasingBound(IndVarStart, RightSCEV, Step, Pred,
- LatchBrExitIdx, &L, SE)) {
- FailureReason = "Unsafe bounds";
- return std::nullopt;
- }
-
- if (LatchBrExitIdx == 0) {
- // We need to decrease the right value unless we have already increased
- // it virtually when we replaced EQ with SLT.
- if (!IncreasedRightValueByOne)
- FixedRightSCEV =
- SE.getMinusSCEV(RightSCEV, SE.getOne(RightSCEV->getType()));
- } else {
- assert(!IncreasedRightValueByOne &&
- "Right value can be increased only for LatchBrExitIdx == 0!");
- }
- }
- BasicBlock *LatchExit = LatchBr->getSuccessor(LatchBrExitIdx);
-
- assert(!L.contains(LatchExit) && "expected an exit block!");
- const DataLayout &DL = Preheader->getModule()->getDataLayout();
- SCEVExpander Expander(SE, DL, "loop-constrainer");
- Instruction *Ins = Preheader->getTerminator();
-
- if (FixedRightSCEV)
- RightValue =
- Expander.expandCodeFor(FixedRightSCEV, FixedRightSCEV->getType(), Ins);
-
- Value *IndVarStartV = Expander.expandCodeFor(IndVarStart, IndVarTy, Ins);
- IndVarStartV->setName("indvar.start");
-
- LoopStructure Result;
-
- Result.Tag = "main";
- Result.Header = Header;
- Result.Latch = Latch;
- Result.LatchBr = LatchBr;
- Result.LatchExit = LatchExit;
- Result.LatchBrExitIdx = LatchBrExitIdx;
- Result.IndVarStart = IndVarStartV;
- Result.IndVarStep = StepCI;
- Result.IndVarBase = LeftValue;
- Result.IndVarIncreasing = IsIncreasing;
- Result.LoopExitAt = RightValue;
- Result.IsSignedPredicate = IsSignedPredicate;
- Result.ExitCountTy = cast<IntegerType>(MaxBETakenCount->getType());
-
- FailureReason = nullptr;
-
- return Result;
-}
-
/// If the type of \p S matches with \p Ty, return \p S. Otherwise, return
/// signed or unsigned extension of \p S to type \p Ty.
static const SCEV *NoopOrExtend(const SCEV *S, Type *Ty, ScalarEvolution &SE,
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index e971c638327bf05..51e8821773c3af3 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -38,6 +38,7 @@ add_llvm_component_library(LLVMTransformUtils
LCSSA.cpp
LibCallsShrinkWrap.cpp
Local.cpp
+ LoopConstrainer.cpp
LoopPeel.cpp
LoopRotationUtils.cpp
LoopSimplify.cpp
diff --git a/llvm/lib/Transforms/Utils/LoopConstrainer.cpp b/llvm/lib/Transforms/Utils/LoopConstrainer.cpp
new file mode 100644
index 000000000000000..802c42a4e386b0b
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/LoopConstrainer.cpp
@@ -0,0 +1,422 @@
+#include "llvm/Transforms/Utils/LoopConstrainer.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+
+using namespace llvm;
+
+static const char *ClonedLoopTag = "loop_constrainer.loop.clone";
+
+#define DEBUG_TYPE "loop-constrainer"
+
+/// Given a loop with an deccreasing induction variable, is it possible to
+/// safely calculate the bounds of a new loop using the given Predicate.
+static bool isSafeDecreasingBound(const SCEV *Start, const SCEV *BoundSCEV,
+ const SCEV *Step, ICmpInst::Predicate Pred,
+ unsigned LatchBrExitIdx, Loop *L,
+ ScalarEvolution &SE) {
+ if (Pred != ICmpInst::ICMP_SLT && Pred != ICmpInst::ICMP_SGT &&
+ Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_UGT)
+ return false;
+
+ if (!SE.isAvailableAtLoopEntry(BoundSCEV, L))
+ return false;
+
+ assert(SE.isKnownNegative(Step) && "expecting negative step");
+
+ LLVM_DEBUG(dbgs() << "isSafeDecreasingBound with:\n");
+ LLVM_DEBUG(dbgs() << "Start: " << *Start << "\n");
+ LLVM_DEBUG(dbgs() << "Step: " << *Step << "\n");
+ LLVM_DEBUG(dbgs() << "BoundSCEV: " << *BoundSCEV << "\n");
+ LLVM_DEBUG(dbgs() << "Pred: " << Pred << "\n");
+ LLVM_DEBUG(dbgs() << "LatchExitBrIdx: " << LatchBrExitIdx << "\n");
+
+ bool IsSigned = ICmpInst::isSigned(Pred);
+ // The predicate that we need to check that the induction variable lies
+ // within bounds.
+ ICmpInst::Predicate BoundPred =
+ IsSigned ? CmpInst::ICMP_SGT : CmpInst::ICMP_UGT;
+
+ if (LatchBrExitIdx == 1)
+ return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, BoundSCEV);
+
+ assert(LatchBrExitIdx == 0 && "LatchBrExitIdx should be either 0 or 1");
+
+ const SCEV *StepPlusOne = SE.getAddExpr(Step, SE.getOne(Step->getType()));
+ unsigned BitWidth = cast<IntegerType>(BoundSCEV->getType())->getBitWidth();
+ APInt Min = IsSigned ? APInt::getSignedMinValue(BitWidth)
+ : APInt::getMinValue(BitWidth);
+ const SCEV *Limit = SE.getMinusSCEV(SE.getConstant(Min), StepPlusOne);
+
+ const SCEV *MinusOne =
+ SE.getMinusSCEV(BoundSCEV, SE.getOne(BoundSCEV->getType()));
+
+ return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, MinusOne) &&
+ SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit);
+}
+
+/// Given a loop with an increasing induction variable, is it possible to
+/// safely calculate the bounds of a new loop using the given Predicate.
+static bool isSafeIncreasingBound(const SCEV *Start, const SCEV *BoundSCEV,
+ const SCEV *Step, ICmpInst::Predicate Pred,
+ unsigned LatchBrExitIdx, Loop *L,
+ ScalarEvolution &SE) {
+ if (Pred != ICmpInst::ICMP_SLT && Pred != ICmpInst::ICMP_SGT &&
+ Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_UGT)
+ return false;
+
+ if (!SE.isAvailableAtLoopEntry(BoundSCEV, L))
+ return false;
+
+ LLVM_DEBUG(dbgs() << "isSafeIncreasingBound with:\n");
+ LLVM_DEBUG(dbgs() << "Start: " << *Start << "\n");
+ LLVM_DEBUG(dbgs() << "Step: " << *Step << "\n");
+ LLVM_DEBUG(dbgs() << "BoundSCEV: " << *BoundSCEV << "\n");
+ LLVM_DEBUG(dbgs() << "Pred: " << Pred << "\n");
+ LLVM_DEBUG(dbgs() << "LatchExitBrIdx: " << LatchBrExitIdx << "\n");
+
+ bool IsSigned = ICmpInst::isSigned(Pred);
+ // The predicate that we need to check that the induction variable lies
+ // within bounds.
+ ICmpInst::Predicate BoundPred =
+ IsSigned ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT;
+
+ if (LatchBrExitIdx == 1)
+ return SE.isLoopEntryGuardedByCond(L, BoundPred, Start, BoundSCEV);
+
+ assert(LatchBrExitIdx == 0 && "LatchBrExitIdx should be 0 or 1");
+
+ const SCEV *StepMinusOne = SE.getMinusSCEV(Step, SE.getOne(Step->getType()));
+ unsigned BitWidth = cast<IntegerType>(BoundSCEV->getType())->getBitWidth();
+ APInt Max = IsSigned ? APInt::getSignedMaxValue(BitWidth)
+ : APInt::getMaxValue(BitWidth);
+ const SCEV *Limit = SE.getMinusSCEV(SE.getConstant(Max), StepMinusOne);
+
+ return (SE.isLoopEntryGuardedByCond(L, BoundPred, Start,
+ SE.getAddExpr(BoundSCEV, Step)) &&
+ SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit));
+}
+
+/// Returns estimate for max latch taken count of the loop of the narrowest
+/// available type. If the latch block has such estimate, it is returned.
+/// Otherwise, we use max exit count of whole loop (that is potentially of wider
+/// type than latch check itself), which is still better than no estimate.
+static const SCEV *getNarrowestLatchMaxTakenCountEstimate(ScalarEvolution &SE,
+ const Loop &L) {
+ const SCEV *FromBlock =
+ SE.getExitCount(&L, L.getLoopLatch(), ScalarEvolution::SymbolicMaximum);
+ if (isa<SCEVCouldNotCompute>(FromBlock))
+ return SE.getSymbolicMaxBackedgeTakenCount(&L);
+ return FromBlock;
+}
+
+std::optional<LoopStructure>
+LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
+ bool AllowUnsignedLatchCond,
+ const char *&FailureReason) {
+ if (!L.isLoopSimplifyForm()) {
+ FailureReason = "loop not in LoopSimplify form";
+ return std::nullopt;
+ }
+
+ BasicBlock *Latch = L.getLoopLatch();
+ assert(Latch && "Simplified loops only have one latch!");
+
+ if (Latch->getTerminator()->getMetadata(ClonedLoopTag)) {
+ FailureReason = "loop has already been cloned";
+ return std::nullopt;
+ }
+
+ if (!L.isLoopExiting(Latch)) {
+ FailureReason = "no loop latch";
+ return std::nullopt;
+ }
+
+ BasicBlock *Header = L.getHeader();
+ BasicBlock *Preheader = L.getLoopPreheader();
+ if (!Preheader) {
+ FailureReason = "no preheader";
+ return std::nullopt;
+ }
+
+ BranchInst *LatchBr = dyn_cast<BranchInst>(Latch->getTerminator());
+ if (!LatchBr || LatchBr->isUnconditional()) {
+ FailureReason = "latch terminator not conditional branch";
+ return std::nullopt;
+ }
+
+ unsigned LatchBrExitIdx = LatchBr->getSuccessor(0) == Header ? 1 : 0;
+
+ ICmpInst *ICI = dyn_cast<ICmpInst>(LatchBr->getCondition());
+ if (!ICI || !isa<IntegerType>(ICI->getOperand(0)->getType())) {
+ FailureReason = "latch terminator branch not conditional on integral icmp";
+ return std::nullopt;
+ }
+
+ const SCEV *MaxBETakenCount = getNarrowestLatchMaxTakenCountEstimate(SE, L);
+ if (isa<SCEVCouldNotCompute>(MaxBETakenCount)) {
+ FailureReason = "could not compute latch count";
+ return std::nullopt;
+ }
+ assert(SE.getLoopDisposition(MaxBETakenCount, &L) ==
+ ScalarEvolution::LoopInvariant &&
+ "loop variant exit count doesn't make sense!");
+
+ ICmpInst::Predicate Pred = ICI->getPredicate();
+ Value *LeftValue = ICI->getOperand(0);
+ const SCEV *LeftSCEV = SE.getSCEV(LeftValue);
+ IntegerType *IndVarTy = cast<IntegerType>(LeftValue->getType());
+
+ Value *RightValue = ICI->getOperand(1);
+ const SCEV *RightSCEV = SE.getSCEV(RightValue);
+
+ // We canonicalize `ICI` such that `LeftSCEV` is an add recurrence.
+ if (!isa<SCEVAddRecExpr>(LeftSCEV)) {
+ if (isa<SCEVAddRecExpr>(RightSCEV)) {
+ std::swap(LeftSCEV, RightSCEV);
+ std::swap(LeftValue, RightValue);
+ Pred = ICmpInst::getSwappedPredicate(Pred);
+ } else {
+ FailureReason = "no add recurrences in the icmp";
+ return std::nullopt;
+ }
+ }
+
+ auto HasNoSignedWrap = [&](const SCEVAddRecExpr *AR) {
+ if (AR->getNoWrapFlags(SCEV::FlagNSW))
+ return true;
+
+ IntegerType *Ty = cast<IntegerType>(AR->getType());
+ IntegerType *WideTy =
+ IntegerType::get(Ty->getContext(), Ty->getBitWidth() * 2);
+
+ const SCEVAddRecExpr *ExtendAfterOp =
+ dyn_cast<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
+ if (ExtendAfterOp) {
+ const SCEV *ExtendedStart = SE.getSignExtendExpr(AR->getStart(), WideTy);
+ const SCEV *ExtendedStep =
+ SE.getSignExtendExpr(AR->getStepRecurrence(SE), WideTy);
+
+ bool NoSignedWrap = ExtendAfterOp->getStart() == ExtendedStart &&
+ ExtendAfterOp->getStepRecurrence(SE) == ExtendedStep;
+
+ if (NoSignedWrap)
+ return true;
+ }
+
+ // We may have proved this when computing the sign extension above.
+ return AR->getNoWrapFlags(SCEV::FlagNSW) != SCEV::FlagAnyWrap;
+ };
+
+ // `ICI` is interpreted as taking the backedge if the *next* value of the
+ // induction variable satisfies some constraint.
+
+ const SCEVAddRecExpr *IndVarBase = cast<SCEVAddRecExpr>(LeftSCEV);
+ if (IndVarBase->getLoop() != &L) {
+ FailureReason = "LHS in cmp is not an AddRec for this loop";
+ return std::nullopt;
+ }
+ if (!IndVarBase->isAffine()) {
+ FailureReason = "LHS in icmp not induction variable";
+ return std::nullopt;
+ }
+ const SCEV *StepRec = IndVarBase->getStepRecurrence(SE);
+ if (!isa<SCEVConstant>(StepRec)) {
+ FailureReason = "LHS in icmp not induction variable";
+ return std::nullopt;
+ }
+ ConstantInt *StepCI = cast<SCEVConstant>(StepRec)->getValue();
+
+ if (ICI->isEquality() && !HasNoSignedWrap(IndVarBase)) {
+ FailureReason = "LHS in icmp needs nsw for equality predicates";
+ return std::nullopt;
+ }
+
+ assert(!StepCI->isZero() && "Zero step?");
+ bool IsIncreasing = !StepCI->isNegative();
+ bool IsSignedPredicate;
+ const SCEV *StartNext = IndVarBase->getStart();
+ const SCEV *Addend = SE.getNegativeSCEV(IndVarBase->getStepRecurrence(SE));
+ const SCEV *IndVarStart = SE.getAddExpr(StartNext, Addend);
+ const SCEV *Step = SE.getSCEV(StepCI);
+
+ const SCEV *FixedRightSCEV = nullptr;
+
+ // If RightValue resides within loop (but still being loop invariant),
+ // regenerate it as preheader.
+ if (auto *I = dyn_cast<Instruction>(RightValue))
+ if (L.contains(I->getParent()))
+ FixedRightSCEV = RightSCEV;
+
+ if (IsIncreasing) {
+ bool DecreasedRightValueByOne = false;
+ if (StepCI->isOne()) {
+ // Try to turn eq/ne predicates to those we can work with.
+ if (Pred == ICmpInst::ICMP_NE && LatchBrExitIdx == 1)
+ // while (++i != len) { while (++i < len) {
+ // ... ---> ...
+ // } }
+ // If both parts are known non-negative, it is profitable to use
+ // unsigned comparison in increasing loop. This allows us to make the
+ // comparison check against "RightSCEV + 1" more optimistic.
+ if (isKnownNonNegativeInLoop(IndVarStart, &L, SE) &&
+ isKnownNonNegativeInLoop(RightSCEV, &L, SE))
+ Pred = ICmpInst::ICMP_ULT;
+ else
+ Pred = ICmpInst::ICMP_SLT;
+ else if (Pred == ICmpInst::ICMP_EQ && LatchBrExitIdx == 0) {
+ // while (true) { while (true) {
+ // if (++i == len) ---> if (++i > len - 1)
+ // break; break;
+ // ... ...
+ // } }
+ if (IndVarBase->getNoWrapFlags(SCEV::FlagNUW) &&
+ cannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/ false)) {
+ Pred = ICmpInst::ICMP_UGT;
+ RightSCEV =
+ SE.getMinusSCEV(RightSCEV, SE.getOne(RightSCEV->getType()));
+ DecreasedRightValueByOne = true;
+ } else if (cannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/ true)) {
+ Pred = ICmpInst::ICMP_SGT;
+ RightSCEV =
+ SE.getMinusSCEV(RightSCEV, SE.getOne(RightSCEV->getType()));
+ DecreasedRightValueByOne = true;
+ }
+ }
+ }
+
+ bool LTPred = (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT);
+ bool GTPred = (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_UGT);
+ bool FoundExpectedPred =
+ (LTPred && LatchBrExitIdx == 1) || (GTPred && LatchBrExitIdx == 0);
+
+ if (!FoundExpectedPred) {
+ FailureReason = "expected icmp slt semantically, found something else";
+ return std::nullopt;
+ }
+
+ IsSignedPredicate = ICmpInst::isSigned(Pred);
+ if (!IsSignedPredicate && !AllowUnsignedLatchCond) {
+ FailureReason = "unsigned latch conditions are explicitly prohibited";
+ return std::nullopt;
+ }
+
+ if (!isSafeIncreasingBound(IndVarStart, RightSCEV, Step, Pred,
+ LatchBrExitIdx, &L, SE)) {
+ FailureReason = "Unsafe loop bounds";
+ return std::nullopt;
+ }
+ if (LatchBrExitIdx == 0) {
+ // We need to increase the right value unless we have already decreased
+ // it virtually when we replaced EQ with SGT.
+ if (!DecreasedRightValueByOne)
+ FixedRightSCEV =
+ SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
+ } else {
+ assert(!DecreasedRightValueByOne &&
+ "Right value can be decreased only for LatchBrExitIdx == 0!");
+ }
+ } else {
+ bool IncreasedRightValueByOne = false;
+ if (StepCI->isMinusOne()) {
+ // Try to turn eq/ne predicates to those we can work with.
+ if (Pred == ICmpInst::ICMP_NE && LatchBrExitIdx == 1)
+ // while (--i != len) { while (--i > len) {
+ // ... ---> ...
+ // } }
+ // We intentionally don't turn the predicate into UGT even if we know
+ // that both operands are non-negative, because it will only pessimize
+ // our check against "RightSCEV - 1".
+ Pred = ICmpInst::ICMP_SGT;
+ else if (Pred == ICmpInst::ICMP_EQ && LatchBrExitIdx == 0) {
+ // while (true) { while (true) {
+ // if (--i == len) ---> if (--i < len + 1)
+ // break; break;
+ // ... ...
+ // } }
+ if (IndVarBase->getNoWrapFlags(SCEV::FlagNUW) &&
+ cannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ false)) {
+ Pred = ICmpInst::ICMP_ULT;
+ RightSCEV = SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
+ IncreasedRightValueByOne = true;
+ } else if (cannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ true)) {
+ Pred = ICmpInst::ICMP_SLT;
+ RightSCEV = SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
+ IncreasedRightValueByOne = true;
+ }
+ }
+ }
+
+ bool LTPred = (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT);
+ bool GTPred = (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_UGT);
+
+ bool FoundExpectedPred =
+ (GTPred && LatchBrExitIdx == 1) || (LTPred && LatchBrExitIdx == 0);
+
+ if (!FoundExpectedPred) {
+ FailureReason = "expected icmp sgt semantically, found something else";
+ return std::nullopt;
+ }
+
+ IsSignedPredicate =
+ Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT;
+
+ if (!IsSignedPredicate && !AllowUnsignedLatchCond) {
+ FailureReason = "unsigned latch conditions are explicitly prohibited";
+ return std::nullopt;
+ }
+
+ if (!isSafeDecreasingBound(IndVarStart, RightSCEV, Step, Pred,
+ LatchBrExitIdx, &L, SE)) {
+ FailureReason = "Unsafe bounds";
+ return std::nullopt;
+ }
+
+ if (LatchBrExitIdx == 0) {
+ // We need to decrease the right value unless we have already increased
+ // it virtually when we replaced EQ with SLT.
+ if (!IncreasedRightValueByOne)
+ FixedRightSCEV =
+ SE.getMinusSCEV(RightSCEV, SE.getOne(RightSCEV->getType()));
+ } else {
+ assert(!IncreasedRightValueByOne &&
+ "Right value can be increased only for LatchBrExitIdx == 0!");
+ }
+ }
+ BasicBlock *LatchExit = LatchBr->getSuccessor(LatchBrExitIdx);
+
+ assert(!L.contains(LatchExit) && "expected an exit block!");
+ const DataLayout &DL = Preheader->getModule()->getDataLayout();
+ SCEVExpander Expander(SE, DL, "loop-constrainer");
+ Instruction *Ins = Preheader->getTerminator();
+
+ if (FixedRightSCEV)
+ RightValue =
+ Expander.expandCodeFor(FixedRightSCEV, FixedRightSCEV->getType(), Ins);
+
+ Value *IndVarStartV = Expander.expandCodeFor(IndVarStart, IndVarTy, Ins);
+ IndVarStartV->setName("indvar.start");
+
+ LoopStructure Result;
+
+ Result.Tag = "main";
+ Result.Header = Header;
+ Result.Latch = Latch;
+ Result.LatchBr = LatchBr;
+ Result.LatchExit = LatchExit;
+ Result.LatchBrExitIdx = LatchBrExitIdx;
+ Result.IndVarStart = IndVarStartV;
+ Result.IndVarStep = StepCI;
+ Result.IndVarBase = LeftValue;
+ Result.IndVarIncreasing = IsIncreasing;
+ Result.LoopExitAt = RightValue;
+ Result.IsSignedPredicate = IsSignedPredicate;
+ Result.ExitCountTy = cast<IntegerType>(MaxBETakenCount->getType());
+
+ FailureReason = nullptr;
+
+ return Result;
+}
+
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index e12f3165c5ecde7..0e76f8286068e43 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -46,6 +46,7 @@ static_library("Utils") {
"LCSSA.cpp",
"LibCallsShrinkWrap.cpp",
"Local.cpp",
+ "LoopConstrainer.cpp"
"LoopPeel.cpp",
"LoopRotationUtils.cpp",
"LoopSimplify.cpp",
>From a5cf18dbe0af52c3f6f404f14d6d2294b12854c2 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Fri, 27 Oct 2023 12:50:29 -0700
Subject: [PATCH 8/8] [NFC] Extract LoopConstrainer form IRCE
Extract LoopConstrainer to a separate file to reuse it outside IRCE.
---
.../llvm/Transforms/Utils/LoopConstrainer.h | 143 ++++
.../Scalar/InductiveRangeCheckElimination.cpp | 617 ------------------
llvm/lib/Transforms/Utils/LoopConstrainer.cpp | 482 ++++++++++++++
3 files changed, 625 insertions(+), 617 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h b/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h
index 9c58c94edd29143..64db907e9a0f412 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopConstrainer.h
@@ -10,15 +10,20 @@
#define LLVM_TRANSFORMS_UTILS_LOOP_CONSTRAINER_H
#include "llvm/Support/Casting.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
#include <optional>
namespace llvm {
class BasicBlock;
class BranchInst;
+class DominatorTree;
class IntegerType;
class Loop;
+class LoopInfo;
+class PHINode;
class ScalarEvolution;
+class SCEV;
class Value;
// Keeps track of the structure of a loop. This is similar to llvm::Loop,
@@ -78,6 +83,144 @@ struct LoopStructure {
static std::optional<LoopStructure>
parseLoopStructure(ScalarEvolution &, Loop &, bool, const char *&);
};
+
+/// This class is used to constrain loops to run within a given iteration space.
+/// The algorithm this class implements is given a Loop and a range [Begin,
+/// End). The algorithm then tries to break out a "main loop" out of the loop
+/// it is given in a way that the "main loop" runs with the induction variable
+/// in a subset of [Begin, End). The algorithm emits appropriate pre and post
+/// loops to run any remaining iterations. The pre loop runs any iterations in
+/// which the induction variable is < Begin, and the post loop runs any
+/// iterations in which the induction variable is >= End.
+class LoopConstrainer {
+public:
+ // Calculated subranges we restrict the iteration space of the main loop to.
+ // See the implementation of `calculateSubRanges' for more details on how
+ // these fields are computed. `LowLimit` is std::nullopt if there is no
+ // restriction on low end of the restricted iteration space of the main loop.
+ // `HighLimit` is std::nullopt if there is no restriction on high end of the
+ // restricted iteration space of the main loop.
+
+ struct SubRanges {
+ std::optional<const SCEV *> LowLimit;
+ std::optional<const SCEV *> HighLimit;
+ };
+
+private:
+ // The representation of a clone of the original loop we started out with.
+ struct ClonedLoop {
+ // The cloned blocks
+ std::vector<BasicBlock *> Blocks;
+
+ // `Map` maps values in the clonee into values in the cloned version
+ ValueToValueMapTy Map;
+
+ // An instance of `LoopStructure` for the cloned loop
+ LoopStructure Structure;
+ };
+
+ // Result of rewriting the range of a loop. See changeIterationSpaceEnd for
+ // more details on what these fields mean.
+ struct RewrittenRangeInfo {
+ BasicBlock *PseudoExit = nullptr;
+ BasicBlock *ExitSelector = nullptr;
+ std::vector<PHINode *> PHIValuesAtPseudoExit;
+ PHINode *IndVarEnd = nullptr;
+
+ RewrittenRangeInfo() = default;
+ };
+
+ // Clone `OriginalLoop' and return the result in CLResult. The IR after
+ // running `cloneLoop' is well formed except for the PHI nodes in CLResult --
+ // the PHI nodes say that there is an incoming edge from `OriginalPreheader`
+ // but there is no such edge.
+ void cloneLoop(ClonedLoop &CLResult, const char *Tag) const;
+
+ // Create the appropriate loop structure needed to describe a cloned copy of
+ // `Original`. The clone is described by `VM`.
+ Loop *createClonedLoopStructure(Loop *Original, Loop *Parent,
+ ValueToValueMapTy &VM, bool IsSubloop);
+
+ // Rewrite the iteration space of the loop denoted by (LS, Preheader). The
+ // iteration space of the rewritten loop ends at ExitLoopAt. The start of the
+ // iteration space is not changed. `ExitLoopAt' is assumed to be slt
+ // `OriginalHeaderCount'.
+ //
+ // If there are iterations left to execute, control is made to jump to
+ // `ContinuationBlock', otherwise they take the normal loop exit. The
+ // returned `RewrittenRangeInfo' object is populated as follows:
+ //
+ // .PseudoExit is a basic block that unconditionally branches to
+ // `ContinuationBlock'.
+ //
+ // .ExitSelector is a basic block that decides, on exit from the loop,
+ // whether to branch to the "true" exit or to `PseudoExit'.
+ //
+ // .PHIValuesAtPseudoExit are PHINodes in `PseudoExit' that compute the value
+ // for each PHINode in the loop header on taking the pseudo exit.
+ //
+ // After changeIterationSpaceEnd, `Preheader' is no longer a legitimate
+ // preheader because it is made to branch to the loop header only
+ // conditionally.
+ RewrittenRangeInfo
+ changeIterationSpaceEnd(const LoopStructure &LS, BasicBlock *Preheader,
+ Value *ExitLoopAt,
+ BasicBlock *ContinuationBlock) const;
+
+ // The loop denoted by `LS' has `OldPreheader' as its preheader. This
+ // function creates a new preheader for `LS' and returns it.
+ BasicBlock *createPreheader(const LoopStructure &LS, BasicBlock *OldPreheader,
+ const char *Tag) const;
+
+ // `ContinuationBlockAndPreheader' was the continuation block for some call to
+ // `changeIterationSpaceEnd' and is the preheader to the loop denoted by `LS'.
+ // This function rewrites the PHI nodes in `LS.Header' to start with the
+ // correct value.
+ void rewriteIncomingValuesForPHIs(
+ LoopStructure &LS, BasicBlock *ContinuationBlockAndPreheader,
+ const LoopConstrainer::RewrittenRangeInfo &RRI) const;
+
+ // Even though we do not preserve any passes at this time, we at least need to
+ // keep the parent loop structure consistent. The `LPPassManager' seems to
+ // verify this after running a loop pass. This function adds the list of
+ // blocks denoted by BBs to this loops parent loop if required.
+ void addToParentLoopIfNeeded(ArrayRef<BasicBlock *> BBs);
+
+ // Some global state.
+ Function &F;
+ LLVMContext &Ctx;
+ ScalarEvolution &SE;
+ DominatorTree &DT;
+ LoopInfo &LI;
+ function_ref<void(Loop *, bool)> LPMAddNewLoop;
+
+ // Information about the original loop we started out with.
+ Loop &OriginalLoop;
+
+ BasicBlock *OriginalPreheader = nullptr;
+
+ // The preheader of the main loop. This may or may not be different from
+ // `OriginalPreheader'.
+ BasicBlock *MainLoopPreheader = nullptr;
+
+ // Type of the range we need to run the main loop in.
+ Type *RangeTy;
+
+ // The structure of the main loop (see comment at the beginning of this class
+ // for a definition)
+ LoopStructure MainLoopStructure;
+
+ SubRanges SR;
+
+public:
+ LoopConstrainer(Loop &L, LoopInfo &LI,
+ function_ref<void(Loop *, bool)> LPMAddNewLoop,
+ const LoopStructure &LS, ScalarEvolution &SE,
+ DominatorTree &DT, Type *T, SubRanges SR);
+
+ // Entry point for the algorithm. Returns true on success.
+ bool run();
+};
} // namespace llvm
#endif // LLVM_TRANSFORMS_UTILS_LOOP_CONSTRAINER_H
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index c17feb0d9d03283..72a20213dfa66fe 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -553,179 +553,6 @@ void InductiveRangeCheck::extractRangeChecksFromBranch(
Checks, Visited);
}
-// Add metadata to the loop L to disable loop optimizations. Callers need to
-// confirm that optimizing loop L is not beneficial.
-static void DisableAllLoopOptsOnLoop(Loop &L) {
- // We do not care about any existing loopID related metadata for L, since we
- // are setting all loop metadata to false.
- LLVMContext &Context = L.getHeader()->getContext();
- // Reserve first location for self reference to the LoopID metadata node.
- MDNode *Dummy = MDNode::get(Context, {});
- MDNode *DisableUnroll = MDNode::get(
- Context, {MDString::get(Context, "llvm.loop.unroll.disable")});
- Metadata *FalseVal =
- ConstantAsMetadata::get(ConstantInt::get(Type::getInt1Ty(Context), 0));
- MDNode *DisableVectorize = MDNode::get(
- Context,
- {MDString::get(Context, "llvm.loop.vectorize.enable"), FalseVal});
- MDNode *DisableLICMVersioning = MDNode::get(
- Context, {MDString::get(Context, "llvm.loop.licm_versioning.disable")});
- MDNode *DisableDistribution= MDNode::get(
- Context,
- {MDString::get(Context, "llvm.loop.distribute.enable"), FalseVal});
- MDNode *NewLoopID =
- MDNode::get(Context, {Dummy, DisableUnroll, DisableVectorize,
- DisableLICMVersioning, DisableDistribution});
- // Set operand 0 to refer to the loop id itself.
- NewLoopID->replaceOperandWith(0, NewLoopID);
- L.setLoopID(NewLoopID);
-}
-
-namespace {
-
-/// This class is used to constrain loops to run within a given iteration space.
-/// The algorithm this class implements is given a Loop and a range [Begin,
-/// End). The algorithm then tries to break out a "main loop" out of the loop
-/// it is given in a way that the "main loop" runs with the induction variable
-/// in a subset of [Begin, End). The algorithm emits appropriate pre and post
-/// loops to run any remaining iterations. The pre loop runs any iterations in
-/// which the induction variable is < Begin, and the post loop runs any
-/// iterations in which the induction variable is >= End.
-class LoopConstrainer {
-public:
- // Calculated subranges we restrict the iteration space of the main loop to.
- // See the implementation of `calculateSubRanges' for more details on how
- // these fields are computed. `LowLimit` is std::nullopt if there is no
- // restriction on low end of the restricted iteration space of the main loop.
- // `HighLimit` is std::nullopt if there is no restriction on high end of the
- // restricted iteration space of the main loop.
-
- struct SubRanges {
- std::optional<const SCEV *> LowLimit;
- std::optional<const SCEV *> HighLimit;
- };
-
-private:
- // The representation of a clone of the original loop we started out with.
- struct ClonedLoop {
- // The cloned blocks
- std::vector<BasicBlock *> Blocks;
-
- // `Map` maps values in the clonee into values in the cloned version
- ValueToValueMapTy Map;
-
- // An instance of `LoopStructure` for the cloned loop
- LoopStructure Structure;
- };
-
- // Result of rewriting the range of a loop. See changeIterationSpaceEnd for
- // more details on what these fields mean.
- struct RewrittenRangeInfo {
- BasicBlock *PseudoExit = nullptr;
- BasicBlock *ExitSelector = nullptr;
- std::vector<PHINode *> PHIValuesAtPseudoExit;
- PHINode *IndVarEnd = nullptr;
-
- RewrittenRangeInfo() = default;
- };
-
- // Clone `OriginalLoop' and return the result in CLResult. The IR after
- // running `cloneLoop' is well formed except for the PHI nodes in CLResult --
- // the PHI nodes say that there is an incoming edge from `OriginalPreheader`
- // but there is no such edge.
- void cloneLoop(ClonedLoop &CLResult, const char *Tag) const;
-
- // Create the appropriate loop structure needed to describe a cloned copy of
- // `Original`. The clone is described by `VM`.
- Loop *createClonedLoopStructure(Loop *Original, Loop *Parent,
- ValueToValueMapTy &VM, bool IsSubloop);
-
- // Rewrite the iteration space of the loop denoted by (LS, Preheader). The
- // iteration space of the rewritten loop ends at ExitLoopAt. The start of the
- // iteration space is not changed. `ExitLoopAt' is assumed to be slt
- // `OriginalHeaderCount'.
- //
- // If there are iterations left to execute, control is made to jump to
- // `ContinuationBlock', otherwise they take the normal loop exit. The
- // returned `RewrittenRangeInfo' object is populated as follows:
- //
- // .PseudoExit is a basic block that unconditionally branches to
- // `ContinuationBlock'.
- //
- // .ExitSelector is a basic block that decides, on exit from the loop,
- // whether to branch to the "true" exit or to `PseudoExit'.
- //
- // .PHIValuesAtPseudoExit are PHINodes in `PseudoExit' that compute the value
- // for each PHINode in the loop header on taking the pseudo exit.
- //
- // After changeIterationSpaceEnd, `Preheader' is no longer a legitimate
- // preheader because it is made to branch to the loop header only
- // conditionally.
- RewrittenRangeInfo
- changeIterationSpaceEnd(const LoopStructure &LS, BasicBlock *Preheader,
- Value *ExitLoopAt,
- BasicBlock *ContinuationBlock) const;
-
- // The loop denoted by `LS' has `OldPreheader' as its preheader. This
- // function creates a new preheader for `LS' and returns it.
- BasicBlock *createPreheader(const LoopStructure &LS, BasicBlock *OldPreheader,
- const char *Tag) const;
-
- // `ContinuationBlockAndPreheader' was the continuation block for some call to
- // `changeIterationSpaceEnd' and is the preheader to the loop denoted by `LS'.
- // This function rewrites the PHI nodes in `LS.Header' to start with the
- // correct value.
- void rewriteIncomingValuesForPHIs(
- LoopStructure &LS, BasicBlock *ContinuationBlockAndPreheader,
- const LoopConstrainer::RewrittenRangeInfo &RRI) const;
-
- // Even though we do not preserve any passes at this time, we at least need to
- // keep the parent loop structure consistent. The `LPPassManager' seems to
- // verify this after running a loop pass. This function adds the list of
- // blocks denoted by BBs to this loops parent loop if required.
- void addToParentLoopIfNeeded(ArrayRef<BasicBlock *> BBs);
-
- // Some global state.
- Function &F;
- LLVMContext &Ctx;
- ScalarEvolution &SE;
- DominatorTree &DT;
- LoopInfo &LI;
- function_ref<void(Loop *, bool)> LPMAddNewLoop;
-
- // Information about the original loop we started out with.
- Loop &OriginalLoop;
-
- BasicBlock *OriginalPreheader = nullptr;
-
- // The preheader of the main loop. This may or may not be different from
- // `OriginalPreheader'.
- BasicBlock *MainLoopPreheader = nullptr;
-
- // Type of the range we need to run the main loop in.
- Type *RangeTy;
-
- // The structure of the main loop (see comment at the beginning of this class
- // for a definition)
- LoopStructure MainLoopStructure;
-
- SubRanges SR;
-
-public:
- LoopConstrainer(Loop &L, LoopInfo &LI,
- function_ref<void(Loop *, bool)> LPMAddNewLoop,
- const LoopStructure &LS, ScalarEvolution &SE,
- DominatorTree &DT, Type *T, SubRanges SR)
- : F(*L.getHeader()->getParent()), Ctx(L.getHeader()->getContext()),
- SE(SE), DT(DT), LI(LI), LPMAddNewLoop(LPMAddNewLoop), OriginalLoop(L),
- RangeTy(T), MainLoopStructure(LS), SR(SR) {}
-
- // Entry point for the algorithm. Returns true on success.
- bool run();
-};
-
-} // end anonymous namespace
-
/// If the type of \p S matches with \p Ty, return \p S. Otherwise, return
/// signed or unsigned extension of \p S to type \p Ty.
static const SCEV *NoopOrExtend(const SCEV *S, Type *Ty, ScalarEvolution &SE,
@@ -818,450 +645,6 @@ calculateSubRanges(ScalarEvolution &SE, const Loop &L,
return Result;
}
-void LoopConstrainer::cloneLoop(LoopConstrainer::ClonedLoop &Result,
- const char *Tag) const {
- for (BasicBlock *BB : OriginalLoop.getBlocks()) {
- BasicBlock *Clone = CloneBasicBlock(BB, Result.Map, Twine(".") + Tag, &F);
- Result.Blocks.push_back(Clone);
- Result.Map[BB] = Clone;
- }
-
- auto GetClonedValue = [&Result](Value *V) {
- assert(V && "null values not in domain!");
- auto It = Result.Map.find(V);
- if (It == Result.Map.end())
- return V;
- return static_cast<Value *>(It->second);
- };
-
- auto *ClonedLatch =
- cast<BasicBlock>(GetClonedValue(OriginalLoop.getLoopLatch()));
- ClonedLatch->getTerminator()->setMetadata(ClonedLoopTag,
- MDNode::get(Ctx, {}));
-
- Result.Structure = MainLoopStructure.map(GetClonedValue);
- Result.Structure.Tag = Tag;
-
- for (unsigned i = 0, e = Result.Blocks.size(); i != e; ++i) {
- BasicBlock *ClonedBB = Result.Blocks[i];
- BasicBlock *OriginalBB = OriginalLoop.getBlocks()[i];
-
- assert(Result.Map[OriginalBB] == ClonedBB && "invariant!");
-
- for (Instruction &I : *ClonedBB)
- RemapInstruction(&I, Result.Map,
- RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-
- // Exit blocks will now have one more predecessor and their PHI nodes need
- // to be edited to reflect that. No phi nodes need to be introduced because
- // the loop is in LCSSA.
-
- for (auto *SBB : successors(OriginalBB)) {
- if (OriginalLoop.contains(SBB))
- continue; // not an exit block
-
- for (PHINode &PN : SBB->phis()) {
- Value *OldIncoming = PN.getIncomingValueForBlock(OriginalBB);
- PN.addIncoming(GetClonedValue(OldIncoming), ClonedBB);
- SE.forgetValue(&PN);
- }
- }
- }
-}
-
-LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
- const LoopStructure &LS, BasicBlock *Preheader, Value *ExitSubloopAt,
- BasicBlock *ContinuationBlock) const {
- // We start with a loop with a single latch:
- //
- // +--------------------+
- // | |
- // | preheader |
- // | |
- // +--------+-----------+
- // | ----------------\
- // | / |
- // +--------v----v------+ |
- // | | |
- // | header | |
- // | | |
- // +--------------------+ |
- // |
- // ..... |
- // |
- // +--------------------+ |
- // | | |
- // | latch >----------/
- // | |
- // +-------v------------+
- // |
- // |
- // | +--------------------+
- // | | |
- // +---> original exit |
- // | |
- // +--------------------+
- //
- // We change the control flow to look like
- //
- //
- // +--------------------+
- // | |
- // | preheader >-------------------------+
- // | | |
- // +--------v-----------+ |
- // | /-------------+ |
- // | / | |
- // +--------v--v--------+ | |
- // | | | |
- // | header | | +--------+ |
- // | | | | | |
- // +--------------------+ | | +-----v-----v-----------+
- // | | | |
- // | | | .pseudo.exit |
- // | | | |
- // | | +-----------v-----------+
- // | | |
- // ..... | | |
- // | | +--------v-------------+
- // +--------------------+ | | | |
- // | | | | | ContinuationBlock |
- // | latch >------+ | | |
- // | | | +----------------------+
- // +---------v----------+ |
- // | |
- // | |
- // | +---------------^-----+
- // | | |
- // +-----> .exit.selector |
- // | |
- // +----------v----------+
- // |
- // +--------------------+ |
- // | | |
- // | original exit <----+
- // | |
- // +--------------------+
-
- RewrittenRangeInfo RRI;
-
- BasicBlock *BBInsertLocation = LS.Latch->getNextNode();
- RRI.ExitSelector = BasicBlock::Create(Ctx, Twine(LS.Tag) + ".exit.selector",
- &F, BBInsertLocation);
- RRI.PseudoExit = BasicBlock::Create(Ctx, Twine(LS.Tag) + ".pseudo.exit", &F,
- BBInsertLocation);
-
- BranchInst *PreheaderJump = cast<BranchInst>(Preheader->getTerminator());
- bool Increasing = LS.IndVarIncreasing;
- bool IsSignedPredicate = LS.IsSignedPredicate;
-
- IRBuilder<> B(PreheaderJump);
- auto NoopOrExt = [&](Value *V) {
- if (V->getType() == RangeTy)
- return V;
- return IsSignedPredicate ? B.CreateSExt(V, RangeTy, "wide." + V->getName())
- : B.CreateZExt(V, RangeTy, "wide." + V->getName());
- };
-
- // EnterLoopCond - is it okay to start executing this `LS'?
- Value *EnterLoopCond = nullptr;
- auto Pred =
- Increasing
- ? (IsSignedPredicate ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT)
- : (IsSignedPredicate ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT);
- Value *IndVarStart = NoopOrExt(LS.IndVarStart);
- EnterLoopCond = B.CreateICmp(Pred, IndVarStart, ExitSubloopAt);
-
- B.CreateCondBr(EnterLoopCond, LS.Header, RRI.PseudoExit);
- PreheaderJump->eraseFromParent();
-
- LS.LatchBr->setSuccessor(LS.LatchBrExitIdx, RRI.ExitSelector);
- B.SetInsertPoint(LS.LatchBr);
- Value *IndVarBase = NoopOrExt(LS.IndVarBase);
- Value *TakeBackedgeLoopCond = B.CreateICmp(Pred, IndVarBase, ExitSubloopAt);
-
- Value *CondForBranch = LS.LatchBrExitIdx == 1
- ? TakeBackedgeLoopCond
- : B.CreateNot(TakeBackedgeLoopCond);
-
- LS.LatchBr->setCondition(CondForBranch);
-
- B.SetInsertPoint(RRI.ExitSelector);
-
- // IterationsLeft - are there any more iterations left, given the original
- // upper bound on the induction variable? If not, we branch to the "real"
- // exit.
- Value *LoopExitAt = NoopOrExt(LS.LoopExitAt);
- Value *IterationsLeft = B.CreateICmp(Pred, IndVarBase, LoopExitAt);
- B.CreateCondBr(IterationsLeft, RRI.PseudoExit, LS.LatchExit);
-
- BranchInst *BranchToContinuation =
- BranchInst::Create(ContinuationBlock, RRI.PseudoExit);
-
- // We emit PHI nodes into `RRI.PseudoExit' that compute the "latest" value of
- // each of the PHI nodes in the loop header. This feeds into the initial
- // value of the same PHI nodes if/when we continue execution.
- for (PHINode &PN : LS.Header->phis()) {
- PHINode *NewPHI = PHINode::Create(PN.getType(), 2, PN.getName() + ".copy",
- BranchToContinuation);
-
- NewPHI->addIncoming(PN.getIncomingValueForBlock(Preheader), Preheader);
- NewPHI->addIncoming(PN.getIncomingValueForBlock(LS.Latch),
- RRI.ExitSelector);
- RRI.PHIValuesAtPseudoExit.push_back(NewPHI);
- }
-
- RRI.IndVarEnd = PHINode::Create(IndVarBase->getType(), 2, "indvar.end",
- BranchToContinuation);
- RRI.IndVarEnd->addIncoming(IndVarStart, Preheader);
- RRI.IndVarEnd->addIncoming(IndVarBase, RRI.ExitSelector);
-
- // The latch exit now has a branch from `RRI.ExitSelector' instead of
- // `LS.Latch'. The PHI nodes need to be updated to reflect that.
- LS.LatchExit->replacePhiUsesWith(LS.Latch, RRI.ExitSelector);
-
- return RRI;
-}
-
-void LoopConstrainer::rewriteIncomingValuesForPHIs(
- LoopStructure &LS, BasicBlock *ContinuationBlock,
- const LoopConstrainer::RewrittenRangeInfo &RRI) const {
- unsigned PHIIndex = 0;
- for (PHINode &PN : LS.Header->phis())
- PN.setIncomingValueForBlock(ContinuationBlock,
- RRI.PHIValuesAtPseudoExit[PHIIndex++]);
-
- LS.IndVarStart = RRI.IndVarEnd;
-}
-
-BasicBlock *LoopConstrainer::createPreheader(const LoopStructure &LS,
- BasicBlock *OldPreheader,
- const char *Tag) const {
- BasicBlock *Preheader = BasicBlock::Create(Ctx, Tag, &F, LS.Header);
- BranchInst::Create(LS.Header, Preheader);
-
- LS.Header->replacePhiUsesWith(OldPreheader, Preheader);
-
- return Preheader;
-}
-
-void LoopConstrainer::addToParentLoopIfNeeded(ArrayRef<BasicBlock *> BBs) {
- Loop *ParentLoop = OriginalLoop.getParentLoop();
- if (!ParentLoop)
- return;
-
- for (BasicBlock *BB : BBs)
- ParentLoop->addBasicBlockToLoop(BB, LI);
-}
-
-Loop *LoopConstrainer::createClonedLoopStructure(Loop *Original, Loop *Parent,
- ValueToValueMapTy &VM,
- bool IsSubloop) {
- Loop &New = *LI.AllocateLoop();
- if (Parent)
- Parent->addChildLoop(&New);
- else
- LI.addTopLevelLoop(&New);
- LPMAddNewLoop(&New, IsSubloop);
-
- // Add all of the blocks in Original to the new loop.
- for (auto *BB : Original->blocks())
- if (LI.getLoopFor(BB) == Original)
- New.addBasicBlockToLoop(cast<BasicBlock>(VM[BB]), LI);
-
- // Add all of the subloops to the new loop.
- for (Loop *SubLoop : *Original)
- createClonedLoopStructure(SubLoop, &New, VM, /* IsSubloop */ true);
-
- return &New;
-}
-
-bool LoopConstrainer::run() {
- BasicBlock *Preheader = OriginalLoop.getLoopPreheader();
- assert(Preheader != nullptr && "precondition!");
-
- OriginalPreheader = Preheader;
- MainLoopPreheader = Preheader;
- bool IsSignedPredicate = MainLoopStructure.IsSignedPredicate;
- bool Increasing = MainLoopStructure.IndVarIncreasing;
- IntegerType *IVTy = cast<IntegerType>(RangeTy);
-
- SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "loop-constrainer");
- Instruction *InsertPt = OriginalPreheader->getTerminator();
-
- // It would have been better to make `PreLoop' and `PostLoop'
- // `std::optional<ClonedLoop>'s, but `ValueToValueMapTy' does not have a copy
- // constructor.
- ClonedLoop PreLoop, PostLoop;
- bool NeedsPreLoop =
- Increasing ? SR.LowLimit.has_value() : SR.HighLimit.has_value();
- bool NeedsPostLoop =
- Increasing ? SR.HighLimit.has_value() : SR.LowLimit.has_value();
-
- Value *ExitPreLoopAt = nullptr;
- Value *ExitMainLoopAt = nullptr;
- const SCEVConstant *MinusOneS =
- cast<SCEVConstant>(SE.getConstant(IVTy, -1, true /* isSigned */));
-
- if (NeedsPreLoop) {
- const SCEV *ExitPreLoopAtSCEV = nullptr;
-
- if (Increasing)
- ExitPreLoopAtSCEV = *SR.LowLimit;
- else if (cannotBeMinInLoop(*SR.HighLimit, &OriginalLoop, SE,
- IsSignedPredicate))
- ExitPreLoopAtSCEV = SE.getAddExpr(*SR.HighLimit, MinusOneS);
- else {
- LLVM_DEBUG(dbgs() << "could not prove no-overflow when computing "
- << "preloop exit limit. HighLimit = "
- << *(*SR.HighLimit) << "\n");
- return false;
- }
-
- if (!Expander.isSafeToExpandAt(ExitPreLoopAtSCEV, InsertPt)) {
- LLVM_DEBUG(dbgs() << "could not prove that it is safe to expand the"
- << " preloop exit limit " << *ExitPreLoopAtSCEV
- << " at block " << InsertPt->getParent()->getName()
- << "\n");
- return false;
- }
-
- ExitPreLoopAt = Expander.expandCodeFor(ExitPreLoopAtSCEV, IVTy, InsertPt);
- ExitPreLoopAt->setName("exit.preloop.at");
- }
-
- if (NeedsPostLoop) {
- const SCEV *ExitMainLoopAtSCEV = nullptr;
-
- if (Increasing)
- ExitMainLoopAtSCEV = *SR.HighLimit;
- else if (cannotBeMinInLoop(*SR.LowLimit, &OriginalLoop, SE,
- IsSignedPredicate))
- ExitMainLoopAtSCEV = SE.getAddExpr(*SR.LowLimit, MinusOneS);
- else {
- LLVM_DEBUG(dbgs() << "could not prove no-overflow when computing "
- << "mainloop exit limit. LowLimit = "
- << *(*SR.LowLimit) << "\n");
- return false;
- }
-
- if (!Expander.isSafeToExpandAt(ExitMainLoopAtSCEV, InsertPt)) {
- LLVM_DEBUG(dbgs() << "could not prove that it is safe to expand the"
- << " main loop exit limit " << *ExitMainLoopAtSCEV
- << " at block " << InsertPt->getParent()->getName()
- << "\n");
- return false;
- }
-
- ExitMainLoopAt = Expander.expandCodeFor(ExitMainLoopAtSCEV, IVTy, InsertPt);
- ExitMainLoopAt->setName("exit.mainloop.at");
- }
-
- // We clone these ahead of time so that we don't have to deal with changing
- // and temporarily invalid IR as we transform the loops.
- if (NeedsPreLoop)
- cloneLoop(PreLoop, "preloop");
- if (NeedsPostLoop)
- cloneLoop(PostLoop, "postloop");
-
- RewrittenRangeInfo PreLoopRRI;
-
- if (NeedsPreLoop) {
- Preheader->getTerminator()->replaceUsesOfWith(MainLoopStructure.Header,
- PreLoop.Structure.Header);
-
- MainLoopPreheader =
- createPreheader(MainLoopStructure, Preheader, "mainloop");
- PreLoopRRI = changeIterationSpaceEnd(PreLoop.Structure, Preheader,
- ExitPreLoopAt, MainLoopPreheader);
- rewriteIncomingValuesForPHIs(MainLoopStructure, MainLoopPreheader,
- PreLoopRRI);
- }
-
- BasicBlock *PostLoopPreheader = nullptr;
- RewrittenRangeInfo PostLoopRRI;
-
- if (NeedsPostLoop) {
- PostLoopPreheader =
- createPreheader(PostLoop.Structure, Preheader, "postloop");
- PostLoopRRI = changeIterationSpaceEnd(MainLoopStructure, MainLoopPreheader,
- ExitMainLoopAt, PostLoopPreheader);
- rewriteIncomingValuesForPHIs(PostLoop.Structure, PostLoopPreheader,
- PostLoopRRI);
- }
-
- BasicBlock *NewMainLoopPreheader =
- MainLoopPreheader != Preheader ? MainLoopPreheader : nullptr;
- BasicBlock *NewBlocks[] = {PostLoopPreheader, PreLoopRRI.PseudoExit,
- PreLoopRRI.ExitSelector, PostLoopRRI.PseudoExit,
- PostLoopRRI.ExitSelector, NewMainLoopPreheader};
-
- // Some of the above may be nullptr, filter them out before passing to
- // addToParentLoopIfNeeded.
- auto NewBlocksEnd =
- std::remove(std::begin(NewBlocks), std::end(NewBlocks), nullptr);
-
- addToParentLoopIfNeeded(ArrayRef(std::begin(NewBlocks), NewBlocksEnd));
-
- DT.recalculate(F);
-
- // We need to first add all the pre and post loop blocks into the loop
- // structures (as part of createClonedLoopStructure), and then update the
- // LCSSA form and LoopSimplifyForm. This is necessary for correctly updating
- // LI when LoopSimplifyForm is generated.
- Loop *PreL = nullptr, *PostL = nullptr;
- if (!PreLoop.Blocks.empty()) {
- PreL = createClonedLoopStructure(&OriginalLoop,
- OriginalLoop.getParentLoop(), PreLoop.Map,
- /* IsSubLoop */ false);
- }
-
- if (!PostLoop.Blocks.empty()) {
- PostL =
- createClonedLoopStructure(&OriginalLoop, OriginalLoop.getParentLoop(),
- PostLoop.Map, /* IsSubLoop */ false);
- }
-
- // This function canonicalizes the loop into Loop-Simplify and LCSSA forms.
- auto CanonicalizeLoop = [&] (Loop *L, bool IsOriginalLoop) {
- formLCSSARecursively(*L, DT, &LI, &SE);
- simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, true);
- // Pre/post loops are slow paths, we do not need to perform any loop
- // optimizations on them.
- if (!IsOriginalLoop)
- DisableAllLoopOptsOnLoop(*L);
- };
- if (PreL)
- CanonicalizeLoop(PreL, false);
- if (PostL)
- CanonicalizeLoop(PostL, false);
- CanonicalizeLoop(&OriginalLoop, true);
-
- /// At this point:
- /// - We've broken a "main loop" out of the loop in a way that the "main loop"
- /// runs with the induction variable in a subset of [Begin, End).
- /// - There is no overflow when computing "main loop" exit limit.
- /// - Max latch taken count of the loop is limited.
- /// It guarantees that induction variable will not overflow iterating in the
- /// "main loop".
- if (isa<OverflowingBinaryOperator>(MainLoopStructure.IndVarBase))
- if (IsSignedPredicate)
- cast<BinaryOperator>(MainLoopStructure.IndVarBase)
- ->setHasNoSignedWrap(true);
- /// TODO: support unsigned predicate.
- /// To add NUW flag we need to prove that both operands of BO are
- /// non-negative. E.g:
- /// ...
- /// %iv.next = add nsw i32 %iv, -1
- /// %cmp = icmp ult i32 %iv.next, %n
- /// br i1 %cmp, label %loopexit, label %loop
- ///
- /// -1 is MAX_UINT in terms of unsigned int. Adding anything but zero will
- /// overflow, therefore NUW flag is not legal here.
-
- return true;
-}
-
/// Computes and returns a range of values for the induction variable (IndVar)
/// in which the range check can be safely elided. If it cannot compute such a
/// range, returns std::nullopt.
diff --git a/llvm/lib/Transforms/Utils/LoopConstrainer.cpp b/llvm/lib/Transforms/Utils/LoopConstrainer.cpp
index 802c42a4e386b0b..ea6d952cfa7d4f3 100644
--- a/llvm/lib/Transforms/Utils/LoopConstrainer.cpp
+++ b/llvm/lib/Transforms/Utils/LoopConstrainer.cpp
@@ -2,6 +2,9 @@
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
@@ -420,3 +423,482 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE, Loop &L,
return Result;
}
+// Add metadata to the loop L to disable loop optimizations. Callers need to
+// confirm that optimizing loop L is not beneficial.
+static void DisableAllLoopOptsOnLoop(Loop &L) {
+ // We do not care about any existing loopID related metadata for L, since we
+ // are setting all loop metadata to false.
+ LLVMContext &Context = L.getHeader()->getContext();
+ // Reserve first location for self reference to the LoopID metadata node.
+ MDNode *Dummy = MDNode::get(Context, {});
+ MDNode *DisableUnroll = MDNode::get(
+ Context, {MDString::get(Context, "llvm.loop.unroll.disable")});
+ Metadata *FalseVal =
+ ConstantAsMetadata::get(ConstantInt::get(Type::getInt1Ty(Context), 0));
+ MDNode *DisableVectorize = MDNode::get(
+ Context,
+ {MDString::get(Context, "llvm.loop.vectorize.enable"), FalseVal});
+ MDNode *DisableLICMVersioning = MDNode::get(
+ Context, {MDString::get(Context, "llvm.loop.licm_versioning.disable")});
+ MDNode *DisableDistribution = MDNode::get(
+ Context,
+ {MDString::get(Context, "llvm.loop.distribute.enable"), FalseVal});
+ MDNode *NewLoopID =
+ MDNode::get(Context, {Dummy, DisableUnroll, DisableVectorize,
+ DisableLICMVersioning, DisableDistribution});
+ // Set operand 0 to refer to the loop id itself.
+ NewLoopID->replaceOperandWith(0, NewLoopID);
+ L.setLoopID(NewLoopID);
+}
+
+LoopConstrainer::LoopConstrainer(Loop &L, LoopInfo &LI,
+ function_ref<void(Loop *, bool)> LPMAddNewLoop,
+ const LoopStructure &LS, ScalarEvolution &SE,
+ DominatorTree &DT, Type *T, SubRanges SR)
+ : F(*L.getHeader()->getParent()), Ctx(L.getHeader()->getContext()), SE(SE),
+ DT(DT), LI(LI), LPMAddNewLoop(LPMAddNewLoop), OriginalLoop(L), RangeTy(T),
+ MainLoopStructure(LS), SR(SR) {}
+
+void LoopConstrainer::cloneLoop(LoopConstrainer::ClonedLoop &Result,
+ const char *Tag) const {
+ for (BasicBlock *BB : OriginalLoop.getBlocks()) {
+ BasicBlock *Clone = CloneBasicBlock(BB, Result.Map, Twine(".") + Tag, &F);
+ Result.Blocks.push_back(Clone);
+ Result.Map[BB] = Clone;
+ }
+
+ auto GetClonedValue = [&Result](Value *V) {
+ assert(V && "null values not in domain!");
+ auto It = Result.Map.find(V);
+ if (It == Result.Map.end())
+ return V;
+ return static_cast<Value *>(It->second);
+ };
+
+ auto *ClonedLatch =
+ cast<BasicBlock>(GetClonedValue(OriginalLoop.getLoopLatch()));
+ ClonedLatch->getTerminator()->setMetadata(ClonedLoopTag,
+ MDNode::get(Ctx, {}));
+
+ Result.Structure = MainLoopStructure.map(GetClonedValue);
+ Result.Structure.Tag = Tag;
+
+ for (unsigned i = 0, e = Result.Blocks.size(); i != e; ++i) {
+ BasicBlock *ClonedBB = Result.Blocks[i];
+ BasicBlock *OriginalBB = OriginalLoop.getBlocks()[i];
+
+ assert(Result.Map[OriginalBB] == ClonedBB && "invariant!");
+
+ for (Instruction &I : *ClonedBB)
+ RemapInstruction(&I, Result.Map,
+ RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
+
+ // Exit blocks will now have one more predecessor and their PHI nodes need
+ // to be edited to reflect that. No phi nodes need to be introduced because
+ // the loop is in LCSSA.
+
+ for (auto *SBB : successors(OriginalBB)) {
+ if (OriginalLoop.contains(SBB))
+ continue; // not an exit block
+
+ for (PHINode &PN : SBB->phis()) {
+ Value *OldIncoming = PN.getIncomingValueForBlock(OriginalBB);
+ PN.addIncoming(GetClonedValue(OldIncoming), ClonedBB);
+ SE.forgetValue(&PN);
+ }
+ }
+ }
+}
+
+LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
+ const LoopStructure &LS, BasicBlock *Preheader, Value *ExitSubloopAt,
+ BasicBlock *ContinuationBlock) const {
+ // We start with a loop with a single latch:
+ //
+ // +--------------------+
+ // | |
+ // | preheader |
+ // | |
+ // +--------+-----------+
+ // | ----------------\
+ // | / |
+ // +--------v----v------+ |
+ // | | |
+ // | header | |
+ // | | |
+ // +--------------------+ |
+ // |
+ // ..... |
+ // |
+ // +--------------------+ |
+ // | | |
+ // | latch >----------/
+ // | |
+ // +-------v------------+
+ // |
+ // |
+ // | +--------------------+
+ // | | |
+ // +---> original exit |
+ // | |
+ // +--------------------+
+ //
+ // We change the control flow to look like
+ //
+ //
+ // +--------------------+
+ // | |
+ // | preheader >-------------------------+
+ // | | |
+ // +--------v-----------+ |
+ // | /-------------+ |
+ // | / | |
+ // +--------v--v--------+ | |
+ // | | | |
+ // | header | | +--------+ |
+ // | | | | | |
+ // +--------------------+ | | +-----v-----v-----------+
+ // | | | |
+ // | | | .pseudo.exit |
+ // | | | |
+ // | | +-----------v-----------+
+ // | | |
+ // ..... | | |
+ // | | +--------v-------------+
+ // +--------------------+ | | | |
+ // | | | | | ContinuationBlock |
+ // | latch >------+ | | |
+ // | | | +----------------------+
+ // +---------v----------+ |
+ // | |
+ // | |
+ // | +---------------^-----+
+ // | | |
+ // +-----> .exit.selector |
+ // | |
+ // +----------v----------+
+ // |
+ // +--------------------+ |
+ // | | |
+ // | original exit <----+
+ // | |
+ // +--------------------+
+
+ RewrittenRangeInfo RRI;
+
+ BasicBlock *BBInsertLocation = LS.Latch->getNextNode();
+ RRI.ExitSelector = BasicBlock::Create(Ctx, Twine(LS.Tag) + ".exit.selector",
+ &F, BBInsertLocation);
+ RRI.PseudoExit = BasicBlock::Create(Ctx, Twine(LS.Tag) + ".pseudo.exit", &F,
+ BBInsertLocation);
+
+ BranchInst *PreheaderJump = cast<BranchInst>(Preheader->getTerminator());
+ bool Increasing = LS.IndVarIncreasing;
+ bool IsSignedPredicate = LS.IsSignedPredicate;
+
+ IRBuilder<> B(PreheaderJump);
+ auto NoopOrExt = [&](Value *V) {
+ if (V->getType() == RangeTy)
+ return V;
+ return IsSignedPredicate ? B.CreateSExt(V, RangeTy, "wide." + V->getName())
+ : B.CreateZExt(V, RangeTy, "wide." + V->getName());
+ };
+
+ // EnterLoopCond - is it okay to start executing this `LS'?
+ Value *EnterLoopCond = nullptr;
+ auto Pred =
+ Increasing
+ ? (IsSignedPredicate ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT)
+ : (IsSignedPredicate ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT);
+ Value *IndVarStart = NoopOrExt(LS.IndVarStart);
+ EnterLoopCond = B.CreateICmp(Pred, IndVarStart, ExitSubloopAt);
+
+ B.CreateCondBr(EnterLoopCond, LS.Header, RRI.PseudoExit);
+ PreheaderJump->eraseFromParent();
+
+ LS.LatchBr->setSuccessor(LS.LatchBrExitIdx, RRI.ExitSelector);
+ B.SetInsertPoint(LS.LatchBr);
+ Value *IndVarBase = NoopOrExt(LS.IndVarBase);
+ Value *TakeBackedgeLoopCond = B.CreateICmp(Pred, IndVarBase, ExitSubloopAt);
+
+ Value *CondForBranch = LS.LatchBrExitIdx == 1
+ ? TakeBackedgeLoopCond
+ : B.CreateNot(TakeBackedgeLoopCond);
+
+ LS.LatchBr->setCondition(CondForBranch);
+
+ B.SetInsertPoint(RRI.ExitSelector);
+
+ // IterationsLeft - are there any more iterations left, given the original
+ // upper bound on the induction variable? If not, we branch to the "real"
+ // exit.
+ Value *LoopExitAt = NoopOrExt(LS.LoopExitAt);
+ Value *IterationsLeft = B.CreateICmp(Pred, IndVarBase, LoopExitAt);
+ B.CreateCondBr(IterationsLeft, RRI.PseudoExit, LS.LatchExit);
+
+ BranchInst *BranchToContinuation =
+ BranchInst::Create(ContinuationBlock, RRI.PseudoExit);
+
+ // We emit PHI nodes into `RRI.PseudoExit' that compute the "latest" value of
+ // each of the PHI nodes in the loop header. This feeds into the initial
+ // value of the same PHI nodes if/when we continue execution.
+ for (PHINode &PN : LS.Header->phis()) {
+ PHINode *NewPHI = PHINode::Create(PN.getType(), 2, PN.getName() + ".copy",
+ BranchToContinuation);
+
+ NewPHI->addIncoming(PN.getIncomingValueForBlock(Preheader), Preheader);
+ NewPHI->addIncoming(PN.getIncomingValueForBlock(LS.Latch),
+ RRI.ExitSelector);
+ RRI.PHIValuesAtPseudoExit.push_back(NewPHI);
+ }
+
+ RRI.IndVarEnd = PHINode::Create(IndVarBase->getType(), 2, "indvar.end",
+ BranchToContinuation);
+ RRI.IndVarEnd->addIncoming(IndVarStart, Preheader);
+ RRI.IndVarEnd->addIncoming(IndVarBase, RRI.ExitSelector);
+
+ // The latch exit now has a branch from `RRI.ExitSelector' instead of
+ // `LS.Latch'. The PHI nodes need to be updated to reflect that.
+ LS.LatchExit->replacePhiUsesWith(LS.Latch, RRI.ExitSelector);
+
+ return RRI;
+}
+
+void LoopConstrainer::rewriteIncomingValuesForPHIs(
+ LoopStructure &LS, BasicBlock *ContinuationBlock,
+ const LoopConstrainer::RewrittenRangeInfo &RRI) const {
+ unsigned PHIIndex = 0;
+ for (PHINode &PN : LS.Header->phis())
+ PN.setIncomingValueForBlock(ContinuationBlock,
+ RRI.PHIValuesAtPseudoExit[PHIIndex++]);
+
+ LS.IndVarStart = RRI.IndVarEnd;
+}
+
+BasicBlock *LoopConstrainer::createPreheader(const LoopStructure &LS,
+ BasicBlock *OldPreheader,
+ const char *Tag) const {
+ BasicBlock *Preheader = BasicBlock::Create(Ctx, Tag, &F, LS.Header);
+ BranchInst::Create(LS.Header, Preheader);
+
+ LS.Header->replacePhiUsesWith(OldPreheader, Preheader);
+
+ return Preheader;
+}
+
+void LoopConstrainer::addToParentLoopIfNeeded(ArrayRef<BasicBlock *> BBs) {
+ Loop *ParentLoop = OriginalLoop.getParentLoop();
+ if (!ParentLoop)
+ return;
+
+ for (BasicBlock *BB : BBs)
+ ParentLoop->addBasicBlockToLoop(BB, LI);
+}
+
+Loop *LoopConstrainer::createClonedLoopStructure(Loop *Original, Loop *Parent,
+ ValueToValueMapTy &VM,
+ bool IsSubloop) {
+ Loop &New = *LI.AllocateLoop();
+ if (Parent)
+ Parent->addChildLoop(&New);
+ else
+ LI.addTopLevelLoop(&New);
+ LPMAddNewLoop(&New, IsSubloop);
+
+ // Add all of the blocks in Original to the new loop.
+ for (auto *BB : Original->blocks())
+ if (LI.getLoopFor(BB) == Original)
+ New.addBasicBlockToLoop(cast<BasicBlock>(VM[BB]), LI);
+
+ // Add all of the subloops to the new loop.
+ for (Loop *SubLoop : *Original)
+ createClonedLoopStructure(SubLoop, &New, VM, /* IsSubloop */ true);
+
+ return &New;
+}
+
+bool LoopConstrainer::run() {
+ BasicBlock *Preheader = OriginalLoop.getLoopPreheader();
+ assert(Preheader != nullptr && "precondition!");
+
+ OriginalPreheader = Preheader;
+ MainLoopPreheader = Preheader;
+ bool IsSignedPredicate = MainLoopStructure.IsSignedPredicate;
+ bool Increasing = MainLoopStructure.IndVarIncreasing;
+ IntegerType *IVTy = cast<IntegerType>(RangeTy);
+
+ SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "loop-constrainer");
+ Instruction *InsertPt = OriginalPreheader->getTerminator();
+
+ // It would have been better to make `PreLoop' and `PostLoop'
+ // `std::optional<ClonedLoop>'s, but `ValueToValueMapTy' does not have a copy
+ // constructor.
+ ClonedLoop PreLoop, PostLoop;
+ bool NeedsPreLoop =
+ Increasing ? SR.LowLimit.has_value() : SR.HighLimit.has_value();
+ bool NeedsPostLoop =
+ Increasing ? SR.HighLimit.has_value() : SR.LowLimit.has_value();
+
+ Value *ExitPreLoopAt = nullptr;
+ Value *ExitMainLoopAt = nullptr;
+ const SCEVConstant *MinusOneS =
+ cast<SCEVConstant>(SE.getConstant(IVTy, -1, true /* isSigned */));
+
+ if (NeedsPreLoop) {
+ const SCEV *ExitPreLoopAtSCEV = nullptr;
+
+ if (Increasing)
+ ExitPreLoopAtSCEV = *SR.LowLimit;
+ else if (cannotBeMinInLoop(*SR.HighLimit, &OriginalLoop, SE,
+ IsSignedPredicate))
+ ExitPreLoopAtSCEV = SE.getAddExpr(*SR.HighLimit, MinusOneS);
+ else {
+ LLVM_DEBUG(dbgs() << "could not prove no-overflow when computing "
+ << "preloop exit limit. HighLimit = "
+ << *(*SR.HighLimit) << "\n");
+ return false;
+ }
+
+ if (!Expander.isSafeToExpandAt(ExitPreLoopAtSCEV, InsertPt)) {
+ LLVM_DEBUG(dbgs() << "could not prove that it is safe to expand the"
+ << " preloop exit limit " << *ExitPreLoopAtSCEV
+ << " at block " << InsertPt->getParent()->getName()
+ << "\n");
+ return false;
+ }
+
+ ExitPreLoopAt = Expander.expandCodeFor(ExitPreLoopAtSCEV, IVTy, InsertPt);
+ ExitPreLoopAt->setName("exit.preloop.at");
+ }
+
+ if (NeedsPostLoop) {
+ const SCEV *ExitMainLoopAtSCEV = nullptr;
+
+ if (Increasing)
+ ExitMainLoopAtSCEV = *SR.HighLimit;
+ else if (cannotBeMinInLoop(*SR.LowLimit, &OriginalLoop, SE,
+ IsSignedPredicate))
+ ExitMainLoopAtSCEV = SE.getAddExpr(*SR.LowLimit, MinusOneS);
+ else {
+ LLVM_DEBUG(dbgs() << "could not prove no-overflow when computing "
+ << "mainloop exit limit. LowLimit = "
+ << *(*SR.LowLimit) << "\n");
+ return false;
+ }
+
+ if (!Expander.isSafeToExpandAt(ExitMainLoopAtSCEV, InsertPt)) {
+ LLVM_DEBUG(dbgs() << "could not prove that it is safe to expand the"
+ << " main loop exit limit " << *ExitMainLoopAtSCEV
+ << " at block " << InsertPt->getParent()->getName()
+ << "\n");
+ return false;
+ }
+
+ ExitMainLoopAt = Expander.expandCodeFor(ExitMainLoopAtSCEV, IVTy, InsertPt);
+ ExitMainLoopAt->setName("exit.mainloop.at");
+ }
+
+ // We clone these ahead of time so that we don't have to deal with changing
+ // and temporarily invalid IR as we transform the loops.
+ if (NeedsPreLoop)
+ cloneLoop(PreLoop, "preloop");
+ if (NeedsPostLoop)
+ cloneLoop(PostLoop, "postloop");
+
+ RewrittenRangeInfo PreLoopRRI;
+
+ if (NeedsPreLoop) {
+ Preheader->getTerminator()->replaceUsesOfWith(MainLoopStructure.Header,
+ PreLoop.Structure.Header);
+
+ MainLoopPreheader =
+ createPreheader(MainLoopStructure, Preheader, "mainloop");
+ PreLoopRRI = changeIterationSpaceEnd(PreLoop.Structure, Preheader,
+ ExitPreLoopAt, MainLoopPreheader);
+ rewriteIncomingValuesForPHIs(MainLoopStructure, MainLoopPreheader,
+ PreLoopRRI);
+ }
+
+ BasicBlock *PostLoopPreheader = nullptr;
+ RewrittenRangeInfo PostLoopRRI;
+
+ if (NeedsPostLoop) {
+ PostLoopPreheader =
+ createPreheader(PostLoop.Structure, Preheader, "postloop");
+ PostLoopRRI = changeIterationSpaceEnd(MainLoopStructure, MainLoopPreheader,
+ ExitMainLoopAt, PostLoopPreheader);
+ rewriteIncomingValuesForPHIs(PostLoop.Structure, PostLoopPreheader,
+ PostLoopRRI);
+ }
+
+ BasicBlock *NewMainLoopPreheader =
+ MainLoopPreheader != Preheader ? MainLoopPreheader : nullptr;
+ BasicBlock *NewBlocks[] = {PostLoopPreheader, PreLoopRRI.PseudoExit,
+ PreLoopRRI.ExitSelector, PostLoopRRI.PseudoExit,
+ PostLoopRRI.ExitSelector, NewMainLoopPreheader};
+
+ // Some of the above may be nullptr, filter them out before passing to
+ // addToParentLoopIfNeeded.
+ auto NewBlocksEnd =
+ std::remove(std::begin(NewBlocks), std::end(NewBlocks), nullptr);
+
+ addToParentLoopIfNeeded(ArrayRef(std::begin(NewBlocks), NewBlocksEnd));
+
+ DT.recalculate(F);
+
+ // We need to first add all the pre and post loop blocks into the loop
+ // structures (as part of createClonedLoopStructure), and then update the
+ // LCSSA form and LoopSimplifyForm. This is necessary for correctly updating
+ // LI when LoopSimplifyForm is generated.
+ Loop *PreL = nullptr, *PostL = nullptr;
+ if (!PreLoop.Blocks.empty()) {
+ PreL = createClonedLoopStructure(&OriginalLoop,
+ OriginalLoop.getParentLoop(), PreLoop.Map,
+ /* IsSubLoop */ false);
+ }
+
+ if (!PostLoop.Blocks.empty()) {
+ PostL =
+ createClonedLoopStructure(&OriginalLoop, OriginalLoop.getParentLoop(),
+ PostLoop.Map, /* IsSubLoop */ false);
+ }
+
+ // This function canonicalizes the loop into Loop-Simplify and LCSSA forms.
+ auto CanonicalizeLoop = [&](Loop *L, bool IsOriginalLoop) {
+ formLCSSARecursively(*L, DT, &LI, &SE);
+ simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, true);
+ // Pre/post loops are slow paths, we do not need to perform any loop
+ // optimizations on them.
+ if (!IsOriginalLoop)
+ DisableAllLoopOptsOnLoop(*L);
+ };
+ if (PreL)
+ CanonicalizeLoop(PreL, false);
+ if (PostL)
+ CanonicalizeLoop(PostL, false);
+ CanonicalizeLoop(&OriginalLoop, true);
+
+ /// At this point:
+ /// - We've broken a "main loop" out of the loop in a way that the "main loop"
+ /// runs with the induction variable in a subset of [Begin, End).
+ /// - There is no overflow when computing "main loop" exit limit.
+ /// - Max latch taken count of the loop is limited.
+ /// It guarantees that induction variable will not overflow iterating in the
+ /// "main loop".
+ if (isa<OverflowingBinaryOperator>(MainLoopStructure.IndVarBase))
+ if (IsSignedPredicate)
+ cast<BinaryOperator>(MainLoopStructure.IndVarBase)
+ ->setHasNoSignedWrap(true);
+ /// TODO: support unsigned predicate.
+ /// To add NUW flag we need to prove that both operands of BO are
+ /// non-negative. E.g:
+ /// ...
+ /// %iv.next = add nsw i32 %iv, -1
+ /// %cmp = icmp ult i32 %iv.next, %n
+ /// br i1 %cmp, label %loopexit, label %loop
+ ///
+ /// -1 is MAX_UINT in terms of unsigned int. Adding anything but zero will
+ /// overflow, therefore NUW flag is not legal here.
+
+ return true;
+}
More information about the llvm-commits
mailing list