[llvm] r270804 - [IRCE] Optimize conjunctions of range checks
Sanjoy Das via llvm-commits
llvm-commits at lists.llvm.org
Wed May 25 17:09:04 PDT 2016
Author: sanjoy
Date: Wed May 25 19:09:02 2016
New Revision: 270804
URL: http://llvm.org/viewvc/llvm-project?rev=270804&view=rev
Log:
[IRCE] Optimize conjunctions of range checks
After this change, we do the expected thing for cases like
```
Check0Passed = /* range check IRCE can optimize */
Check1Passed = /* range check IRCE can optimize */
if (!(Check0Passed && Check1Passed))
throw_Exception();
```
Added:
llvm/trunk/test/Transforms/IRCE/conjunctive-checks.ll
Modified:
llvm/trunk/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
Modified: llvm/trunk/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp?rev=270804&r1=270803&r2=270804&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp (original)
+++ llvm/trunk/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp Wed May 25 19:09:02 2016
@@ -125,8 +125,10 @@ class InductiveRangeCheck {
ScalarEvolution &SE, Value *&Index,
Value *&Length);
- static Optional<InductiveRangeCheck>
- parseRangeCheckFromCond(Loop *L, ScalarEvolution &SE, Use &ConditionUse);
+ static void
+ extractRangeChecksFromCond(Loop *L, ScalarEvolution &SE, Use &ConditionUse,
+ SmallVectorImpl<InductiveRangeCheck> &Checks,
+ SmallPtrSetImpl<Value *> &Visited);
InductiveRangeCheck()
: Offset(nullptr), Scale(nullptr), Length(nullptr),
@@ -189,10 +191,15 @@ public:
Optional<Range> computeSafeIterationSpace(ScalarEvolution &SE,
const SCEVAddRecExpr *IndVar) const;
- /// Create an inductive range check out of BI if possible, else return None.
- static Optional<InductiveRangeCheck> create(BranchInst *BI, Loop *L,
- ScalarEvolution &SE,
- BranchProbabilityInfo &BPI);
+ /// Parse out a set of inductive range checks from \p BI and append them to \p
+ /// Checks.
+ ///
+ /// NB! There may be conditions feeding into \p BI that aren't inductive range
+ /// checks, and hence don't end up in \p Checks.
+ static void
+ extractRangeChecksFromBranch(BranchInst *BI, Loop *L, ScalarEvolution &SE,
+ BranchProbabilityInfo &BPI,
+ SmallVectorImpl<InductiveRangeCheck> &Checks);
};
class InductiveRangeCheckElimination : public LoopPass {
@@ -312,54 +319,64 @@ InductiveRangeCheck::parseRangeCheckICmp
llvm_unreachable("default clause returns!");
}
-/// Parses an arbitrary condition into an inductive range check.
-Optional<InductiveRangeCheck>
-InductiveRangeCheck::parseRangeCheckFromCond(Loop *L, ScalarEvolution &SE,
- Use &ConditionUse) {
+void InductiveRangeCheck::extractRangeChecksFromCond(
+ Loop *L, ScalarEvolution &SE, Use &ConditionUse,
+ SmallVectorImpl<InductiveRangeCheck> &Checks,
+ SmallPtrSetImpl<Value *> &Visited) {
using namespace llvm::PatternMatch;
Value *Condition = ConditionUse.get();
+ if (!Visited.insert(Condition).second)
+ return;
- Value *Length, *Index;
- InductiveRangeCheck::RangeCheckKind RCKind;
-
- Value *A = nullptr;
- Value *B = nullptr;
-
- if (match(Condition, m_And(m_Value(A), m_Value(B)))) {
- Value *IndexA = nullptr, *IndexB = nullptr;
- Value *LengthA = nullptr, *LengthB = nullptr;
- ICmpInst *ICmpA = dyn_cast<ICmpInst>(A), *ICmpB = dyn_cast<ICmpInst>(B);
+ if (match(Condition, m_And(m_Value(), m_Value()))) {
+ SmallVector<InductiveRangeCheck, 8> SubChecks;
+ extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(0),
+ SubChecks, Visited);
+ extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(1),
+ SubChecks, Visited);
+
+ if (SubChecks.size() == 2) {
+ // Handle a special case where we know how to merge two checks separately
+ // checking the upper and lower bounds into a full range check.
+ const auto &RChkA = SubChecks[0];
+ const auto &RChkB = SubChecks[1];
+ if ((RChkA.Length == RChkB.Length || !RChkA.Length || !RChkB.Length) &&
+ RChkA.Offset == RChkB.Offset && RChkA.Scale == RChkB.Scale) {
+
+ // If RChkA.Kind == RChkB.Kind then we just found two identical checks.
+ // But if one of them is a RANGE_CHECK_LOWER and the other is a
+ // RANGE_CHECK_UPPER (only possibility if they're different) then
+ // together they form a RANGE_CHECK_BOTH.
+ SubChecks[0].Kind =
+ (InductiveRangeCheck::RangeCheckKind)(RChkA.Kind | RChkB.Kind);
+ SubChecks[0].Length = RChkA.Length ? RChkA.Length : RChkB.Length;
+ SubChecks[0].CheckUse = &ConditionUse;
- if (!ICmpA || !ICmpB)
- return None;
+ // We updated one of the checks in place, now erase the other.
+ SubChecks.pop_back();
+ }
+ }
- auto RCKindA = parseRangeCheckICmp(L, ICmpA, SE, IndexA, LengthA);
- auto RCKindB = parseRangeCheckICmp(L, ICmpB, SE, IndexB, LengthB);
+ Checks.insert(Checks.end(), SubChecks.begin(), SubChecks.end());
+ return;
+ }
- if (RCKindA == InductiveRangeCheck::RANGE_CHECK_UNKNOWN ||
- RCKindB == InductiveRangeCheck::RANGE_CHECK_UNKNOWN ||
- IndexA != IndexB ||
- (LengthA != nullptr && LengthB != nullptr && LengthA != LengthB))
- return None;
+ ICmpInst *ICI = dyn_cast<ICmpInst>(Condition);
+ if (!ICI)
+ return;
- Index = IndexA;
- Length = LengthA == nullptr ? LengthB : LengthA;
- RCKind = (InductiveRangeCheck::RangeCheckKind)(RCKindA | RCKindB);
- } else if (ICmpInst *ICI = dyn_cast<ICmpInst>(Condition)) {
- RCKind = parseRangeCheckICmp(L, ICI, SE, Index, Length);
- if (RCKind == InductiveRangeCheck::RANGE_CHECK_UNKNOWN)
- return None;
- } else {
- return None;
- }
+ Value *Length = nullptr, *Index;
+ auto RCKind = parseRangeCheckICmp(L, ICI, SE, Index, Length);
+ if (RCKind == InductiveRangeCheck::RANGE_CHECK_UNKNOWN)
+ return;
const auto *IndexAddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Index));
bool IsAffineIndex =
IndexAddRec && (IndexAddRec->getLoop() == L) && IndexAddRec->isAffine();
if (!IsAffineIndex)
- return None;
+ return;
InductiveRangeCheck IRC;
IRC.Length = Length;
@@ -367,23 +384,24 @@ InductiveRangeCheck::parseRangeCheckFrom
IRC.Scale = IndexAddRec->getStepRecurrence(SE);
IRC.CheckUse = &ConditionUse;
IRC.Kind = RCKind;
- return IRC;
+ Checks.push_back(IRC);
}
-Optional<InductiveRangeCheck>
-InductiveRangeCheck::create(BranchInst *BI, Loop *L, ScalarEvolution &SE,
- BranchProbabilityInfo &BPI) {
+void InductiveRangeCheck::extractRangeChecksFromBranch(
+ BranchInst *BI, Loop *L, ScalarEvolution &SE, BranchProbabilityInfo &BPI,
+ SmallVectorImpl<InductiveRangeCheck> &Checks) {
if (BI->isUnconditional() || BI->getParent() == L->getLoopLatch())
- return None;
+ return;
BranchProbability LikelyTaken(15, 16);
if (BPI.getEdgeProbability(BI->getParent(), (unsigned)0) < LikelyTaken)
- return None;
+ return;
- return InductiveRangeCheck::parseRangeCheckFromCond(L, SE,
- BI->getOperandUse(0));
+ SmallPtrSet<Value *, 8> Visited;
+ InductiveRangeCheck::extractRangeChecksFromCond(L, SE, BI->getOperandUse(0),
+ Checks, Visited);
}
namespace {
@@ -1370,8 +1388,8 @@ bool InductiveRangeCheckElimination::run
for (auto BBI : L->getBlocks())
if (BranchInst *TBI = dyn_cast<BranchInst>(BBI->getTerminator()))
- if (auto MaybeIRC = InductiveRangeCheck::create(TBI, L, SE, BPI))
- RangeChecks.push_back(*MaybeIRC);
+ InductiveRangeCheck::extractRangeChecksFromBranch(TBI, L, SE, BPI,
+ RangeChecks);
if (RangeChecks.empty())
return false;
Added: llvm/trunk/test/Transforms/IRCE/conjunctive-checks.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/IRCE/conjunctive-checks.ll?rev=270804&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/IRCE/conjunctive-checks.ll (added)
+++ llvm/trunk/test/Transforms/IRCE/conjunctive-checks.ll Wed May 25 19:09:02 2016
@@ -0,0 +1,99 @@
+; RUN: opt -S -irce < %s | FileCheck %s
+
+define void @f_0(i32 *%arr, i32 *%a_len_ptr, i32 %n, i1* %cond_buf) {
+; CHECK-LABEL: @f_0(
+
+; CHECK-LABEL: loop.preheader:
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_safe_range_end:[^ ]+]] = sub i32 3, %len
+; CHECK: [[not_exit_main_loop_at_hiclamp_cmp:[^ ]+]] = icmp sgt i32 [[not_n]], [[not_safe_range_end]]
+; CHECK: [[not_exit_main_loop_at_hiclamp:[^ ]+]] = select i1 [[not_exit_main_loop_at_hiclamp_cmp]], i32 [[not_n]], i32 [[not_safe_range_end]]
+; CHECK: [[exit_main_loop_at_hiclamp:[^ ]+]] = sub i32 -1, [[not_exit_main_loop_at_hiclamp]]
+; CHECK: [[exit_main_loop_at_loclamp_cmp:[^ ]+]] = icmp sgt i32 [[exit_main_loop_at_hiclamp]], 0
+; CHECK: [[exit_main_loop_at_loclamp:[^ ]+]] = select i1 [[exit_main_loop_at_loclamp_cmp]], i32 [[exit_main_loop_at_hiclamp]], i32 0
+; CHECK: [[enter_main_loop:[^ ]+]] = icmp slt i32 0, [[exit_main_loop_at_loclamp]]
+; CHECK: br i1 [[enter_main_loop]], label %loop, label %main.pseudo.exit
+
+ entry:
+ %len = load i32, i32* %a_len_ptr, !range !0
+ %first.itr.check = icmp sgt i32 %n, 0
+ br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+ %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+ %idx.next = add i32 %idx, 1
+ %idx.for.abc = add i32 %idx, 4
+ %abc.actual = icmp slt i32 %idx.for.abc, %len
+ %cond = load volatile i1, i1* %cond_buf
+ %abc = and i1 %cond, %abc.actual
+ br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+; CHECK: loop:
+; CHECK: %cond = load volatile i1, i1* %cond_buf
+; CHECK: %abc = and i1 %cond, true
+; CHECK: br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+ %addr = getelementptr i32, i32* %arr, i32 %idx.for.abc
+ store i32 0, i32* %addr
+ %next = icmp slt i32 %idx.next, %n
+ br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+ ret void
+
+ exit:
+ ret void
+}
+
+define void @f_1(
+ i32* %arr_a, i32* %a_len_ptr, i32* %arr_b, i32* %b_len_ptr, i32 %n) {
+; CHECK-LABEL: @f_1(
+
+; CHECK-LABEL: loop.preheader:
+; CHECK: [[not_len_b:[^ ]+]] = sub i32 -1, %len.b
+; CHECK: [[not_len_a:[^ ]+]] = sub i32 -1, %len.a
+; CHECK: [[smax_not_len_cond:[^ ]+]] = icmp sgt i32 [[not_len_b]], [[not_len_a]]
+; CHECK: [[smax_not_len:[^ ]+]] = select i1 [[smax_not_len_cond]], i32 [[not_len_b]], i32 [[not_len_a]]
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_upper_limit_cond_loclamp:[^ ]+]] = icmp sgt i32 [[smax_not_len]], [[not_n]]
+; CHECK: [[not_upper_limit_loclamp:[^ ]+]] = select i1 [[not_upper_limit_cond_loclamp]], i32 [[smax_not_len]], i32 [[not_n]]
+; CHECK: [[upper_limit_loclamp:[^ ]+]] = sub i32 -1, [[not_upper_limit_loclamp]]
+; CHECK: [[upper_limit_cmp:[^ ]+]] = icmp sgt i32 [[upper_limit_loclamp]], 0
+; CHECK: [[upper_limit:[^ ]+]] = select i1 [[upper_limit_cmp]], i32 [[upper_limit_loclamp]], i32 0
+
+ entry:
+ %len.a = load i32, i32* %a_len_ptr, !range !0
+ %len.b = load i32, i32* %b_len_ptr, !range !0
+ %first.itr.check = icmp sgt i32 %n, 0
+ br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+ %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+ %idx.next = add i32 %idx, 1
+ %abc.a = icmp slt i32 %idx, %len.a
+ %abc.b = icmp slt i32 %idx, %len.b
+ %abc = and i1 %abc.a, %abc.b
+ br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+; CHECK: loop:
+; CHECK: %abc = and i1 true, true
+; CHECK: br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+ %addr.a = getelementptr i32, i32* %arr_a, i32 %idx
+ store i32 0, i32* %addr.a
+ %addr.b = getelementptr i32, i32* %arr_b, i32 %idx
+ store i32 -1, i32* %addr.b
+ %next = icmp slt i32 %idx.next, %n
+ br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+ ret void
+
+ exit:
+ ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
More information about the llvm-commits
mailing list