[llvm] [llvm][ipsccp/sccp] Strengthen range analysis in SCCPSolver (PR #111716)
Grigory Pastukhov via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 9 10:12:37 PDT 2024
https://github.com/grigorypas created https://github.com/llvm/llvm-project/pull/111716
Resolves #108906
Currently, when a PHI node is visited excessively, the corresponding ValueLatticeElement is marked as "overdefined". This pull request proposes a more refined approach that leverages additional information from the control flow to improve the precision of the analysis.
Key ideas are:
1. **GuaranteedBoundsPropagator:** When a variable x passes through a conditional branch (e.g., x < 10), we can now infer that x will always be less than 10 in that branch and use that instead of "overdefined".
2. **MonotonicityTracker:** If a variable x undergoes a series of transformations within a loop that are either always increasing or always decreasing, we can determine a lower or upper bound for x based on its initial value.
cc: @dcci, @nikic, @fhahn
>From b1f03c83ea37587320d835cb83c601b95ebd49c4 Mon Sep 17 00:00:00 2001
From: Grigory Pastukhov <gpastukhov at meta.com>
Date: Sun, 6 Oct 2024 18:38:54 -0700
Subject: [PATCH 1/4] Implementation of GuaranteedBoundsPropagator
---
llvm/lib/Transforms/Utils/SCCPSolver.cpp | 158 ++++++++++++++++++++++-
1 file changed, 155 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index 101d60525f4161..0bd5f2e66d942b 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -383,6 +383,78 @@ void SCCPSolver::inferArgAttributes() const {
}
}
+// GuaranteedBoundsPropagator is a class that propagates
+// guaranteed bounds for values. Typically, a ValueLatticeElement
+// associated with a frequently visited Phi node is marked as "overdefined" to
+// prevent excessive iterations. However, GuaranteedBoundsPropagator enhances
+// this by propagating guaranteed bounds up to the Phi node, potentially
+// improving precision.
+// Consider a scenario where a variable 'x' is evaluated in a branch with the
+// condition 'x < 10'. In this case, we can confidently assert that 'x' will not
+// exceed 10. GuaranteedBoundsPropagator leverages this information by
+// propagating such guaranteed bounds up to the relevant Phi node. If all
+// incoming values to the Phi node have guaranteed bounds, the union of these
+// bounds will represent the guaranteed bounds for the Phi node itself. Once
+// these bounds are established for the Phi node, they can be propagated further
+// to other values that depend on this Phi node.
+// However, if not all incoming branches to the Phi node have been explored or
+// are active, the bounds for the Phi node cannot be fully guaranteed. In such
+// cases, the propagator may still apply the best available bounds to the Phi
+// node instead of marking it as overdefined. These bounds remain valid unless
+// new branches become active. If any active incoming branches lack guaranteed
+// bounds, the Phi node's state need to be adjusted to overdefined.
+class GuaranteedBoundsPropagator {
+ DenseMap<Value *, ConstantRange> GuaranteedBounds;
+
+public:
+ std::optional<ConstantRange> getGuaranteedBounds(Value *V) {
+ if (!V->getType()->isIntegerTy())
+ return {};
+ if (Constant *C = dyn_cast<Constant>(V))
+ return C->toConstantRange();
+ auto It = GuaranteedBounds.find(V);
+ if (It == GuaranteedBounds.end())
+ return {};
+ auto &Range = It->second;
+ if (Range.isFullSet())
+ return {};
+ return Range;
+ }
+
+ void insertOrUpdate(Value *V, const ConstantRange &CR) {
+ if (CR.isFullSet())
+ return;
+ auto It = GuaranteedBounds.find(V);
+ if (It == GuaranteedBounds.end()) {
+ GuaranteedBounds.insert({V, CR});
+ } else {
+ It->second = CR;
+ }
+ }
+
+ // If ImposedCR is not full set, then we update guaranteed bounds
+ // of OutputValue. If in addition InputValue has guaranteed bounds,
+ // we update the guaranteed bounds of OutputValue to be the intersection
+ // of the two.
+ void processConditionalBranch(Value *OutputValue, Value *InputValue,
+ const ConstantRange &ImposedCR);
+
+ // Updates the guaranteed bounds of the corresponding value if the
+ // operands have guaranteed bounds.
+ void processBinaryOp(Instruction *I);
+
+ // If guaranteed bounds from all incoming edges are known, the union of all
+ // of the bounds is returned. The flag \p IsBoundGuaranteed is set to true.
+ // If all the incoming edges were not explored yet, but the ones that were
+ // all have guaranteed bounds, the union of the bounds is returned and the
+ // flag \p IsBoundGuaranteed is set to false. If some of the incoming edges
+ // do not have guaranteed bounds (or we failed to calculate union),
+ // the function returns std::nullopt.
+ std::optional<ConstantRange> processPhiNode(
+ PHINode *PN,
+ const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches);
+};
+
/// Helper class for SCCPSolver. This implements the instruction visitor and
/// holds all the state.
class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
@@ -450,6 +522,8 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
DenseMap<Value *, SmallSetVector<User *, 2>> AdditionalUsers;
+ GuaranteedBoundsPropagator BoundsPropagator;
+
LLVMContext &Ctx;
private:
@@ -1255,6 +1329,7 @@ void SCCPInstVisitor::visitPHINode(PHINode &PN) {
return (void)markOverdefined(&PN);
unsigned NumActiveIncoming = 0;
+ SmallVector<Value *, 8> IncomingValuesFromActiveBranches;
// Look at all of the executable operands of the PHI node. If any of them
// are overdefined, the PHI becomes overdefined as well. If they are all
@@ -1265,7 +1340,7 @@ void SCCPInstVisitor::visitPHINode(PHINode &PN) {
for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
if (!isEdgeFeasible(PN.getIncomingBlock(i), PN.getParent()))
continue;
-
+ IncomingValuesFromActiveBranches.push_back(PN.getIncomingValue(i));
ValueLatticeElement IV = getValueState(PN.getIncomingValue(i));
PhiState.mergeIn(IV);
NumActiveIncoming++;
@@ -1273,6 +1348,27 @@ void SCCPInstVisitor::visitPHINode(PHINode &PN) {
break;
}
+ // If we have visited this PHI node too many times, we first check
+ // if there is a known bounds we could use. If not, the state will
+ // be maked as overdefined.
+ auto OptionalBestBounds =
+ BoundsPropagator.processPhiNode(&PN, IncomingValuesFromActiveBranches);
+ auto &OldState = getValueState(&PN);
+ if (OptionalBestBounds && PhiState.isConstantRange() &&
+ OldState.isConstantRange()) {
+ ConstantRange OldStateRange = OldState.getConstantRange();
+ ConstantRange NewStateRange = PhiState.getConstantRange();
+ if (OldStateRange != NewStateRange &&
+ OldState.getNumRangeExtensions() > NumActiveIncoming) {
+ PhiState = ValueLatticeElement::getRange(*OptionalBestBounds);
+ mergeInValue(&PN, PhiState);
+ ValueLatticeElement &PhiStateRef = getValueState(&PN);
+ PhiStateRef.setNumRangeExtensions(
+ std::max(NumActiveIncoming, PhiStateRef.getNumRangeExtensions()));
+ return;
+ }
+ }
+
// We allow up to 1 range extension per active incoming value and one
// additional extension. Note that we manually adjust the number of range
// extensions to match the number of active incoming values. This helps to
@@ -1280,7 +1376,7 @@ void SCCPInstVisitor::visitPHINode(PHINode &PN) {
// incoming values are equal.
mergeInValue(&PN, PhiState,
ValueLatticeElement::MergeOptions().setMaxWidenSteps(
- NumActiveIncoming + 1));
+ NumActiveIncoming + 2));
ValueLatticeElement &PhiStateRef = getValueState(&PN);
PhiStateRef.setNumRangeExtensions(
std::max(NumActiveIncoming, PhiStateRef.getNumRangeExtensions()));
@@ -1585,6 +1681,8 @@ void SCCPInstVisitor::visitBinaryOperator(Instruction &I) {
R = A.overflowingBinaryOp(BO->getOpcode(), B, OBO->getNoWrapKind());
else
R = A.binaryOp(BO->getOpcode(), B);
+
+ BoundsPropagator.processBinaryOp(&I);
mergeInValue(&I, ValueLatticeElement::getRange(R));
// TODO: Currently we do not exploit special values that produce something
@@ -1880,9 +1978,12 @@ void SCCPInstVisitor::handleCallResult(CallBase &CB) {
ConstantRange::getFull(DL.getTypeSizeInBits(CopyOf->getType()));
// Get the range imposed by the condition.
- if (CondVal.isConstantRange())
+ if (CondVal.isConstantRange()) {
ImposedCR = ConstantRange::makeAllowedICmpRegion(
Pred, CondVal.getConstantRange());
+ if (BoundsPropagator.getGuaranteedBounds(OtherOp))
+ BoundsPropagator.processConditionalBranch(&CB, CopyOf, ImposedCR);
+ }
// Combine range info for the original value with the new range from the
// condition.
@@ -2252,3 +2353,54 @@ void SCCPSolver::markFunctionUnreachable(Function *F) {
void SCCPSolver::visit(Instruction *I) { Visitor->visit(I); }
void SCCPSolver::visitCall(CallInst &I) { Visitor->visitCall(I); }
+
+void GuaranteedBoundsPropagator::processConditionalBranch(
+ Value *OutputValue, Value *InputValue, const ConstantRange &ImposedCR) {
+ auto OptionalInputValueBounds = getGuaranteedBounds(InputValue);
+ if (OptionalInputValueBounds)
+ insertOrUpdate(OutputValue,
+ ImposedCR.intersectWith(*OptionalInputValueBounds));
+ else
+ insertOrUpdate(OutputValue, ImposedCR);
+}
+
+void GuaranteedBoundsPropagator::processBinaryOp(Instruction *I) {
+ auto *BO = cast<BinaryOperator>(I);
+ assert(BO && "Expected binary op");
+ auto OptionalLHSBounds = getGuaranteedBounds(BO->getOperand(0));
+ auto OptionalRHSBounds = getGuaranteedBounds(BO->getOperand(1));
+ if (!OptionalLHSBounds || !OptionalRHSBounds)
+ return;
+ ConstantRange R =
+ ConstantRange::getEmpty(I->getType()->getScalarSizeInBits());
+ if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(BO))
+ R = OptionalLHSBounds->overflowingBinaryOp(
+ BO->getOpcode(), *OptionalRHSBounds, OBO->getNoWrapKind());
+ else
+ R = OptionalLHSBounds->binaryOp(BO->getOpcode(), *OptionalRHSBounds);
+ insertOrUpdate(I, R);
+}
+
+std::optional<ConstantRange> GuaranteedBoundsPropagator::processPhiNode(
+ PHINode *PN,
+ const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches) {
+ auto OptionalExistingBounds = getGuaranteedBounds(PN);
+ if (OptionalExistingBounds)
+ return *OptionalExistingBounds;
+
+ ConstantRange R =
+ ConstantRange::getEmpty(PN->getType()->getScalarSizeInBits());
+ for (Value *IncomingValue : IncomingValuesFromActiveBranches) {
+ auto OptionalIncomingBounds = getGuaranteedBounds(IncomingValue);
+ if (!OptionalIncomingBounds)
+ return {};
+ // TODO: Handle disjoint ranges in the future, if needed.
+ auto OptionalUnion = R.exactUnionWith(*OptionalIncomingBounds);
+ if (!OptionalUnion)
+ return {};
+ R = *OptionalUnion;
+ }
+ if (PN->getNumIncomingValues() == IncomingValuesFromActiveBranches.size())
+ insertOrUpdate(PN, R);
+ return R;
+}
>From 634ff93559ae0b9569518a15645863fd0e11c444 Mon Sep 17 00:00:00 2001
From: Grigory Pastukhov <gpastukhov at meta.com>
Date: Sun, 6 Oct 2024 18:39:52 -0700
Subject: [PATCH 2/4] Adding regression test file
---
llvm/test/Transforms/SCCP/loop-removal.ll | 43 +++++++++++++++++++++++
1 file changed, 43 insertions(+)
create mode 100644 llvm/test/Transforms/SCCP/loop-removal.ll
diff --git a/llvm/test/Transforms/SCCP/loop-removal.ll b/llvm/test/Transforms/SCCP/loop-removal.ll
new file mode 100644
index 00000000000000..05f7a096734ef5
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/loop-removal.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=ipsccp -S | FileCheck %s
+
+define i32 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: init:
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL:.*]]
+; CHECK: outer.loop.control:
+; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[INIT:%.*]] ], [ [[X_OUTER:%.*]], [[OUTER_LOOP_INC:%.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp slt i32 [[X_0]], 10
+; CHECK-NEXT: br i1 [[TMP0]], label %[[INNER_LOOP_CONTROL:.*]], label %[[EXIT:.*]]
+; CHECK: inner.loop.control:
+; CHECK-NEXT: br label [[OUTER_LOOP_INC]]
+; CHECK: outer.loop.inc:
+; CHECK-NEXT: [[X_OUTER]] = add nsw i32 [[X_0]], 2
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[X_0]]
+;
+init:
+ br label %outer.loop.control
+
+outer.loop.control: ; preds = %init, %outer.loop.inc
+ %x.0 = phi i32 [ 0, %init ], [ %x.outer, %outer.loop.inc ]
+ %0 = icmp slt i32 %x.0, 10
+ br i1 %0, label %inner.loop.control, label %exit
+
+inner.loop.control: ; preds = %outer.loop.control, %inner.loop.body
+ %x.1 = phi i32 [ %x.0, %outer.loop.control ], [ %x.inner, %inner.loop.body ]
+ %1 = icmp sgt i32 %x.1, 20
+ br i1 %1, label %inner.loop.body, label %outer.loop.inc
+
+inner.loop.body: ; preds = %inner.loop.control
+ %x.inner = sub nsw i32 %x.1, 1
+ br label %inner.loop.control
+
+outer.loop.inc: ; preds = %inner.loop.control
+ %x.outer = add nsw i32 %x.1, 2
+ br label %outer.loop.control
+
+exit: ; preds = %1
+ ret i32 %x.0
+}
>From e059c6ab2811d0924933d7972ddad2fe4cd35acd Mon Sep 17 00:00:00 2001
From: Grigory Pastukhov <gpastukhov at meta.com>
Date: Sun, 6 Oct 2024 19:57:30 -0700
Subject: [PATCH 3/4] Fixed failing test: the pass now eliminates more branches
---
llvm/test/Transforms/SCCP/undef-resolve.ll | 13 +++----------
1 file changed, 3 insertions(+), 10 deletions(-)
diff --git a/llvm/test/Transforms/SCCP/undef-resolve.ll b/llvm/test/Transforms/SCCP/undef-resolve.ll
index 8bb2baa82dce36..7f8842f31f037f 100644
--- a/llvm/test/Transforms/SCCP/undef-resolve.ll
+++ b/llvm/test/Transforms/SCCP/undef-resolve.ll
@@ -38,20 +38,13 @@ define i32 @test2() nounwind readnone ssp {
; CHECK-NEXT: br label [[CONTROL_US:%.*]]
; CHECK: bb3.us:
; CHECK-NEXT: br label [[CONTROL_OUTER_US]]
-; CHECK: bb0.us:
-; CHECK-NEXT: br label [[CONTROL_US]]
; CHECK: control.us:
-; CHECK-NEXT: [[SWITCHCOND_0_US]] = phi i32 [ [[A_0_PH_US]], [[BB0_US:%.*]] ], [ [[SWITCHCOND_0_PH_US]], [[CONTROL_OUTER_US]] ]
-; CHECK-NEXT: switch i32 [[SWITCHCOND_0_US]], label [[CONTROL_OUTER_LOOPEXIT_US_LCSSA_US:%.*]] [
-; CHECK-NEXT: i32 0, label [[BB0_US]]
-; CHECK-NEXT: i32 1, label [[BB1_US_LCSSA_US:%.*]]
-; CHECK-NEXT: i32 3, label [[BB3_US]]
+; CHECK-NEXT: switch i32 [[SWITCHCOND_0_PH_US]], label [[CONTROL_OUTER_LOOPEXIT_US_LCSSA_US:%.*]] [
; CHECK-NEXT: i32 4, label [[BB4_US_LCSSA_US:%.*]]
+; CHECK-NEXT: i32 3, label [[BB3_US]]
; CHECK-NEXT: ]
; CHECK: control.outer.loopexit.us-lcssa.us:
; CHECK-NEXT: br label [[CONTROL_OUTER_LOOPEXIT]]
-; CHECK: bb1.us-lcssa.us:
-; CHECK-NEXT: br label [[BB1:%.*]]
; CHECK: bb4.us-lcssa.us:
; CHECK-NEXT: br label [[BB4:%.*]]
; CHECK: control.outer:
@@ -79,7 +72,7 @@ define i32 @test2() nounwind readnone ssp {
; CHECK: bb0:
; CHECK-NEXT: br label [[CONTROL]]
; CHECK: bb1.us-lcssa:
-; CHECK-NEXT: br label [[BB1]]
+; CHECK-NEXT: br label [[BB1:%.*]]
; CHECK: bb1:
; CHECK-NEXT: ret i32 0
;
>From c67a05934b127af6faf383e84f594d7196ff951e Mon Sep 17 00:00:00 2001
From: Grigory Pastukhov <gpastukhov at meta.com>
Date: Tue, 8 Oct 2024 22:26:10 -0700
Subject: [PATCH 4/4] Implemented tracking monotonicity
---
llvm/lib/Transforms/Utils/SCCPSolver.cpp | 223 ++++++++++++++++--
.../test/Transforms/SCCP/conditions-ranges.ll | 4 +-
llvm/test/Transforms/SCCP/loop-removal.ll | 166 ++++++++++++-
llvm/test/Transforms/SCCP/widening.ll | 4 +-
4 files changed, 375 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index 0bd5f2e66d942b..f76638a8c56fd6 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -455,6 +455,53 @@ class GuaranteedBoundsPropagator {
const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches);
};
+// The MonotonicityTracker class evaluates whether a variable x, which is
+// subject to various functional transformations within a loop, maintains a
+// consistent pattern of monotonicity. Monotonicity, in this context, refers
+// to the property of a sequence where its elements consistently increase
+// or decrease. When a variable x enters a loop and is transformed by a series
+// of operations before it merges back at a PHI node, this class checks if all
+// transformations applied to x preserve the same type of monotonicity
+// (either increasing or decreasing). If the monotonicity is consistent and
+// increasing, for instance, it can be inferred that the value of x at the end
+// of the loop will always be greater than its initial value at the start
+// of the loop.
+class MonotonicityTracker {
+public:
+ enum class Monotonicity {
+ UNKNOWN,
+ CONSTANT,
+ INCREASING,
+ DECREASING,
+ };
+
+ Monotonicity getMonotonicity(Value *V) const {
+ if (isa<Constant>(V))
+ return Monotonicity::CONSTANT;
+ auto It = MonotonicityMap.find(V);
+ if (It == MonotonicityMap.end())
+ return Monotonicity::UNKNOWN;
+ return It->second;
+ }
+
+ void processSSACopyIntrinsic(CallBase &CB) {
+ Value *Op = CB.getOperand(0);
+ Monotonicity OpMonotonicity = getMonotonicity(Op);
+ if (OpMonotonicity == Monotonicity::UNKNOWN)
+ return;
+ MonotonicityMap[&CB] = OpMonotonicity;
+ }
+
+ void processPhiNode(
+ PHINode &PN,
+ const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches);
+
+ void processBinaryOp(Instruction &I);
+
+private:
+ DenseMap<Value *, Monotonicity> MonotonicityMap;
+};
+
/// Helper class for SCCPSolver. This implements the instruction visitor and
/// holds all the state.
class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
@@ -524,6 +571,8 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
GuaranteedBoundsPropagator BoundsPropagator;
+ MonotonicityTracker MonotonicityTrackerObj;
+
LLVMContext &Ctx;
private:
@@ -989,6 +1038,11 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
}
Invalidated.clear();
}
+
+ bool mergeInWithBoundsOverrideForPhi(
+ PHINode &PN,
+ const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches,
+ ValueLatticeElement &PhiState);
};
} // namespace llvm
@@ -1297,6 +1351,65 @@ bool SCCPInstVisitor::isEdgeFeasible(BasicBlock *From, BasicBlock *To) const {
return KnownFeasibleEdges.count(Edge(From, To));
}
+bool SCCPInstVisitor::mergeInWithBoundsOverrideForPhi(
+ PHINode &PN,
+ const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches,
+ ValueLatticeElement &PhiState) {
+ if (!PhiState.isConstantRange(false))
+ return false;
+ auto &OldState = getValueState(&PN);
+ if (!OldState.isConstantRange(false))
+ return false;
+ ConstantRange OldStateRange = OldState.getConstantRange();
+ ConstantRange NewStateRange = PhiState.getConstantRange();
+ unsigned NumActiveIncoming = IncomingValuesFromActiveBranches.size();
+ if (OldStateRange == NewStateRange ||
+ OldState.getNumRangeExtensions() <= NumActiveIncoming)
+ return false;
+
+ // At this point we visited Phi node too many times and need
+ // to extend the range
+ MonotonicityTracker::Monotonicity Monotonicity =
+ MonotonicityTrackerObj.getMonotonicity(&PN);
+
+ // Monotonicity cannot be CONSTANT, otherwise the range should not
+ // have changed
+ if (Monotonicity == MonotonicityTracker::Monotonicity::INCREASING &&
+ OldStateRange.getUpper() == NewStateRange.getUpper()) {
+ mergeInValue(&PN, PhiState);
+ return true;
+ }
+
+ if (Monotonicity == MonotonicityTracker::Monotonicity::DECREASING &&
+ OldStateRange.getLower() == NewStateRange.getLower()) {
+ mergeInValue(&PN, PhiState);
+ return true;
+ }
+
+ auto OptionalBestBounds =
+ BoundsPropagator.processPhiNode(&PN, IncomingValuesFromActiveBranches);
+ if (!OptionalBestBounds)
+ return false;
+ if (Monotonicity == MonotonicityTracker::Monotonicity::INCREASING) {
+ mergeInValue(
+ &PN, ValueLatticeElement::getRange(ConstantRange(
+ NewStateRange.getLower(), OptionalBestBounds->getUpper())));
+ return true;
+ }
+
+ if (Monotonicity == MonotonicityTracker::Monotonicity::DECREASING &&
+ OptionalBestBounds) {
+ mergeInValue(
+ &PN, ValueLatticeElement::getRange(ConstantRange(
+ OptionalBestBounds->getLower(), NewStateRange.getUpper())));
+ return true;
+ }
+
+ PhiState = ValueLatticeElement::getRange(*OptionalBestBounds);
+ mergeInValue(&PN, PhiState);
+ return true;
+}
+
// visit Implementations - Something changed in this instruction, either an
// operand made a transition, or the instruction is newly executable. Change
// the value type of I to reflect these changes if appropriate. This method
@@ -1348,26 +1461,14 @@ void SCCPInstVisitor::visitPHINode(PHINode &PN) {
break;
}
+ MonotonicityTrackerObj.processPhiNode(PN, IncomingValuesFromActiveBranches);
+
// If we have visited this PHI node too many times, we first check
// if there is a known bounds we could use. If not, the state will
// be maked as overdefined.
- auto OptionalBestBounds =
- BoundsPropagator.processPhiNode(&PN, IncomingValuesFromActiveBranches);
- auto &OldState = getValueState(&PN);
- if (OptionalBestBounds && PhiState.isConstantRange() &&
- OldState.isConstantRange()) {
- ConstantRange OldStateRange = OldState.getConstantRange();
- ConstantRange NewStateRange = PhiState.getConstantRange();
- if (OldStateRange != NewStateRange &&
- OldState.getNumRangeExtensions() > NumActiveIncoming) {
- PhiState = ValueLatticeElement::getRange(*OptionalBestBounds);
- mergeInValue(&PN, PhiState);
- ValueLatticeElement &PhiStateRef = getValueState(&PN);
- PhiStateRef.setNumRangeExtensions(
- std::max(NumActiveIncoming, PhiStateRef.getNumRangeExtensions()));
- return;
- }
- }
+ if (mergeInWithBoundsOverrideForPhi(PN, IncomingValuesFromActiveBranches,
+ PhiState))
+ return;
// We allow up to 1 range extension per active incoming value and one
// additional extension. Note that we manually adjust the number of range
@@ -1642,6 +1743,8 @@ void SCCPInstVisitor::visitBinaryOperator(Instruction &I) {
if (V1State.isOverdefined() && V2State.isOverdefined())
return (void)markOverdefined(&I);
+ MonotonicityTrackerObj.processBinaryOp(I);
+
// If either of the operands is a constant, try to fold it to a constant.
// TODO: Use information from notconstant better.
if ((V1State.isConstant() || V2State.isConstant())) {
@@ -1947,6 +2050,7 @@ void SCCPInstVisitor::handleCallResult(CallBase &CB) {
if (auto *II = dyn_cast<IntrinsicInst>(&CB)) {
if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
+ MonotonicityTrackerObj.processSSACopyIntrinsic(CB);
if (ValueState[&CB].isOverdefined())
return;
@@ -2404,3 +2508,88 @@ std::optional<ConstantRange> GuaranteedBoundsPropagator::processPhiNode(
insertOrUpdate(PN, R);
return R;
}
+
+void MonotonicityTracker::processPhiNode(
+ PHINode &PN,
+ const SmallVector<Value *, 8> &IncomingValuesFromActiveBranches) {
+ Monotonicity MI = Monotonicity::UNKNOWN;
+ // If all incoming values have the same monotonicity, then the phi
+ // node will be assinged the same monotonicity as well. However, if
+ // there are different monotonicities or unknown for some incoming values,
+ // then the phi node will be assigned UNKNOWN.
+ for (Value *IncomingValue : IncomingValuesFromActiveBranches) {
+ auto IncomingMonotonicity = getMonotonicity(IncomingValue);
+ if (IncomingMonotonicity == Monotonicity::UNKNOWN) {
+ MI = IncomingMonotonicity;
+ break;
+ }
+ if (IncomingMonotonicity == Monotonicity::CONSTANT &&
+ MI != Monotonicity::UNKNOWN)
+ continue;
+ if (MI != IncomingMonotonicity && MI != Monotonicity::UNKNOWN &&
+ MI != Monotonicity::CONSTANT) {
+ MI = Monotonicity::UNKNOWN;
+ break;
+ }
+ if (MI == Monotonicity::UNKNOWN || MI == Monotonicity::CONSTANT)
+ MI = IncomingMonotonicity;
+ }
+ MonotonicityMap[&PN] = MI;
+}
+
+static bool increasingOrConst(MonotonicityTracker::Monotonicity M) {
+ return M == MonotonicityTracker::Monotonicity::INCREASING ||
+ M == MonotonicityTracker::Monotonicity::CONSTANT;
+}
+
+static bool decreasingOrConst(MonotonicityTracker::Monotonicity M) {
+ return M == MonotonicityTracker::Monotonicity::DECREASING ||
+ M == MonotonicityTracker::Monotonicity::CONSTANT;
+}
+
+void MonotonicityTracker::processBinaryOp(Instruction &I) {
+ auto *BO = dyn_cast<OverflowingBinaryOperator>(&I);
+ if (!I.getType()->isIntegerTy())
+ return;
+ if (!BO || !(BO->hasNoSignedWrap() || BO->hasNoUnsignedWrap()))
+ return;
+ if (isa<Constant>(BO->getOperand(0)) && isa<Constant>(BO->getOperand(1))) {
+ MonotonicityMap[&I] = Monotonicity::CONSTANT;
+ return;
+ }
+ if (!isa<Constant>(BO->getOperand(0)) && !isa<Constant>(BO->getOperand(1))) {
+ MonotonicityMap[&I] = Monotonicity::UNKNOWN;
+ return;
+ }
+ // Here one operand must be a constant and the other must be a variable.
+ ConstantInt *C;
+ Value *NonConstOp;
+ if (isa<Constant>(BO->getOperand(0))) {
+ C = cast<ConstantInt>(BO->getOperand(0));
+ NonConstOp = BO->getOperand(1);
+ } else {
+ C = cast<ConstantInt>(BO->getOperand(1));
+ NonConstOp = BO->getOperand(0);
+ }
+ Monotonicity NonConstOpMonotonicity = getMonotonicity(NonConstOp);
+ if (BO->getOpcode() == Instruction::Add) {
+ if (increasingOrConst(NonConstOpMonotonicity) && !C->isNegative()) {
+ MonotonicityMap[&I] = Monotonicity::INCREASING;
+ return;
+ } else if (decreasingOrConst(NonConstOpMonotonicity) && C->isNegative()) {
+ MonotonicityMap[&I] = Monotonicity::DECREASING;
+ return;
+ }
+ }
+ // Handle subtraction only if it is in the form of "x - constant".
+ if (BO->getOpcode() == Instruction::Sub &&
+ !isa<Constant>(BO->getOperand(0))) {
+ if (decreasingOrConst(NonConstOpMonotonicity) && !C->isNegative()) {
+ MonotonicityMap[&I] = Monotonicity::DECREASING;
+ return;
+ } else if (increasingOrConst(NonConstOpMonotonicity) && C->isNegative()) {
+ MonotonicityMap[&I] = Monotonicity::INCREASING;
+ return;
+ }
+ }
+}
diff --git a/llvm/test/Transforms/SCCP/conditions-ranges.ll b/llvm/test/Transforms/SCCP/conditions-ranges.ll
index bb3764160f7246..829598c7b14f61 100644
--- a/llvm/test/Transforms/SCCP/conditions-ranges.ll
+++ b/llvm/test/Transforms/SCCP/conditions-ranges.ll
@@ -675,7 +675,7 @@ define void @loop_1() {
; CHECK: for.cond11:
; CHECK-NEXT: br label [[FOR_COND_CLEANUP13]]
; CHECK: for.cond.cleanup13:
-; CHECK-NEXT: [[INC27]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: [[INC27]] = add nuw nsw i32 [[I_0]], 1
; CHECK-NEXT: br label [[FOR_COND]]
;
entry:
@@ -718,7 +718,7 @@ define void @loop() {
; CHECK-NEXT: [[CMP12:%.*]] = icmp slt i32 [[J_0]], 2
; CHECK-NEXT: br i1 [[CMP12]], label [[FOR_BODY14]], label [[FOR_COND_CLEANUP13]]
; CHECK: for.cond.cleanup13:
-; CHECK-NEXT: [[INC27]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: [[INC27]] = add nuw nsw i32 [[I_0]], 1
; CHECK-NEXT: br label [[FOR_COND]]
; CHECK: for.body14:
; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[J_0]], 1
diff --git a/llvm/test/Transforms/SCCP/loop-removal.ll b/llvm/test/Transforms/SCCP/loop-removal.ll
index 05f7a096734ef5..5a933d22742c9f 100644
--- a/llvm/test/Transforms/SCCP/loop-removal.ll
+++ b/llvm/test/Transforms/SCCP/loop-removal.ll
@@ -12,7 +12,7 @@ define i32 @foo() {
; CHECK: inner.loop.control:
; CHECK-NEXT: br label [[OUTER_LOOP_INC]]
; CHECK: outer.loop.inc:
-; CHECK-NEXT: [[X_OUTER]] = add nsw i32 [[X_0]], 2
+; CHECK-NEXT: [[X_OUTER]] = add nuw nsw i32 [[X_0]], 2
; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL]]
; CHECK: exit:
; CHECK-NEXT: ret i32 [[X_0]]
@@ -41,3 +41,167 @@ outer.loop.inc: ; preds = %inner.
exit: ; preds = %1
ret i32 %x.0
}
+
+define i32 @test_monotonicity_add() {
+; CHECK-LABEL: @test_monotonicity_add(
+; CHECK-NEXT: init:
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL:.*]]
+; CHECK: outer.loop.control:
+; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[INIT:%.*]] ], [ [[X_OUTER:%.*]], [[OUTER_LOOP_INC:%.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp slt i32 [[X_0]], 10
+; CHECK-NEXT: br i1 [[TMP0]], label %[[INNER_LOOP_CONTROL:.*]], label %[[EXIT:.*]]
+; CHECK: inner.loop.control:
+; CHECK-NEXT: br label [[OUTER_LOOP_INC]]
+; CHECK: outer.loop.inc:
+; CHECK-NEXT: [[X_OUTER]] = add nuw nsw i32 [[X_0]], 2
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[X_0]]
+;
+init:
+ br label %outer.loop.control
+
+outer.loop.control: ; preds = %init, %outer.loop.inc
+ %x.0 = phi i32 [ 0, %init ], [ %x.outer, %outer.loop.inc ]
+ %0 = icmp slt i32 %x.0, 10
+ br i1 %0, label %inner.loop.control, label %exit
+
+inner.loop.control: ; preds = %outer.loop.control, %inner.loop.body
+ %x.1 = phi i32 [ %x.0, %outer.loop.control ], [ %x.inner, %inner.loop.body ]
+ %1 = icmp slt i32 %x.1, -5
+ br i1 %1, label %inner.loop.body, label %outer.loop.inc
+
+inner.loop.body: ; preds = %inner.loop.control
+ %x.inner = sub nsw i32 %x.1, 1
+ br label %inner.loop.control
+
+outer.loop.inc: ; preds = %inner.loop.control
+ %x.outer = add nsw i32 %x.1, 2
+ br label %outer.loop.control
+
+exit: ; preds = %1
+ ret i32 %x.0
+}
+
+define i32 @test_monotonicity_add_negative() {
+; CHECK-LABEL: @test_monotonicity_add_negative(
+; CHECK-NEXT: init:
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL:.*]]
+; CHECK: outer.loop.control:
+; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 100, [[INIT:%.*]] ], [ [[X_OUTER:%.*]], [[OUTER_LOOP_INC:%.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i32 [[X_0]], 10
+; CHECK-NEXT: br i1 [[TMP0]], label %[[INNER_LOOP_CONTROL:.*]], label %[[EXIT:.*]]
+; CHECK: inner.loop.control:
+; CHECK-NEXT: br label [[OUTER_LOOP_INC]]
+; CHECK: outer.loop.inc:
+; CHECK-NEXT: [[X_OUTER]] = add nsw i32 [[X_0]], -2
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[X_0]]
+;
+init:
+ br label %outer.loop.control
+
+outer.loop.control: ; preds = %init, %outer.loop.inc
+ %x.0 = phi i32 [ 100, %init ], [ %x.outer, %outer.loop.inc ]
+ %0 = icmp sgt i32 %x.0, 10
+ br i1 %0, label %inner.loop.control, label %exit
+
+inner.loop.control: ; preds = %outer.loop.control, %inner.loop.body
+ %x.1 = phi i32 [ %x.0, %outer.loop.control ], [ %x.inner, %inner.loop.body ]
+ %1 = icmp sgt i32 %x.1, 150
+ br i1 %1, label %inner.loop.body, label %outer.loop.inc
+
+inner.loop.body: ; preds = %inner.loop.control
+ %x.inner = sub nsw i32 %x.1, 1
+ br label %inner.loop.control
+
+outer.loop.inc: ; preds = %inner.loop.control
+ %x.outer = add nsw i32 %x.1, -2
+ br label %outer.loop.control
+
+exit: ; preds = %1
+ ret i32 %x.0
+}
+
+define i32 @test_monotonicity_sub() {
+; CHECK-LABEL: @test_monotonicity_sub(
+; CHECK-NEXT: init:
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL:.*]]
+; CHECK: outer.loop.control:
+; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 100, [[INIT:%.*]] ], [ [[X_OUTER:%.*]], [[OUTER_LOOP_INC:%.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i32 [[X_0]], 10
+; CHECK-NEXT: br i1 [[TMP0]], label %[[INNER_LOOP_CONTROL:.*]], label %[[EXIT:.*]]
+; CHECK: inner.loop.control:
+; CHECK-NEXT: br label [[OUTER_LOOP_INC]]
+; CHECK: outer.loop.inc:
+; CHECK-NEXT: [[X_OUTER]] = sub nuw nsw i32 [[X_0]], 2
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[X_0]]
+;
+init:
+ br label %outer.loop.control
+
+outer.loop.control: ; preds = %init, %outer.loop.inc
+ %x.0 = phi i32 [ 100, %init ], [ %x.outer, %outer.loop.inc ]
+ %0 = icmp sgt i32 %x.0, 10
+ br i1 %0, label %inner.loop.control, label %exit
+
+inner.loop.control: ; preds = %outer.loop.control, %inner.loop.body
+ %x.1 = phi i32 [ %x.0, %outer.loop.control ], [ %x.inner, %inner.loop.body ]
+ %1 = icmp sgt i32 %x.1, 150
+ br i1 %1, label %inner.loop.body, label %outer.loop.inc
+
+inner.loop.body: ; preds = %inner.loop.control
+ %x.inner = sub nsw i32 %x.1, 1
+ br label %inner.loop.control
+
+outer.loop.inc: ; preds = %inner.loop.control
+ %x.outer = sub nsw i32 %x.1, 2
+ br label %outer.loop.control
+
+exit: ; preds = %1
+ ret i32 %x.0
+}
+
+define i32 @test_monotonicity_sub_negative() {
+; CHECK-LABEL: @test_monotonicity_sub_negative(
+; CHECK-NEXT: init:
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL:.*]]
+; CHECK: outer.loop.control:
+; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 0, [[INIT:%.*]] ], [ [[X_OUTER:%.*]], [[OUTER_LOOP_INC:%.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp slt i32 [[X_0]], 10
+; CHECK-NEXT: br i1 [[TMP0]], label %[[INNER_LOOP_CONTROL:.*]], label %[[EXIT:.*]]
+; CHECK: inner.loop.control:
+; CHECK-NEXT: br label [[OUTER_LOOP_INC]]
+; CHECK: outer.loop.inc:
+; CHECK-NEXT: [[X_OUTER]] = sub nsw i32 [[X_0]], -2
+; CHECK-NEXT: br label %[[OUTER_LOOP_CONTROL]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[X_0]]
+;
+init:
+ br label %outer.loop.control
+
+outer.loop.control: ; preds = %init, %outer.loop.inc
+ %x.0 = phi i32 [ 0, %init ], [ %x.outer, %outer.loop.inc ]
+ %0 = icmp slt i32 %x.0, 10
+ br i1 %0, label %inner.loop.control, label %exit
+
+inner.loop.control: ; preds = %outer.loop.control, %inner.loop.body
+ %x.1 = phi i32 [ %x.0, %outer.loop.control ], [ %x.inner, %inner.loop.body ]
+ %1 = icmp slt i32 %x.1, -5
+ br i1 %1, label %inner.loop.body, label %outer.loop.inc
+
+inner.loop.body: ; preds = %inner.loop.control
+ %x.inner = sub nsw i32 %x.1, 1
+ br label %inner.loop.control
+
+outer.loop.inc: ; preds = %inner.loop.control
+ %x.outer = sub nsw i32 %x.1, -2
+ br label %outer.loop.control
+
+exit: ; preds = %1
+ ret i32 %x.0
+}
diff --git a/llvm/test/Transforms/SCCP/widening.ll b/llvm/test/Transforms/SCCP/widening.ll
index 144c3fd54e75b6..3679a93ac385c5 100644
--- a/llvm/test/Transforms/SCCP/widening.ll
+++ b/llvm/test/Transforms/SCCP/widening.ll
@@ -406,7 +406,7 @@ define void @loop_with_header_2(i32 %x) {
; IPSCCP-NEXT: br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
; IPSCCP: loop.body:
; IPSCCP-NEXT: call void @use(i1 true)
-; IPSCCP-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IPSCCP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; IPSCCP-NEXT: br label [[LOOP_HEADER]]
; IPSCCP: exit:
; IPSCCP-NEXT: ret void
@@ -706,7 +706,7 @@ define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
; IPSCCP-NEXT: [[TMP28:%.*]] = icmp eq i32 [[TMP27]], [[TMP3]]
; IPSCCP-NEXT: br i1 [[TMP28]], label [[BB31]], label [[BB29]]
; IPSCCP: bb29:
-; IPSCCP-NEXT: [[TMP30]] = add nsw i32 [[TMP11]], 1
+; IPSCCP-NEXT: [[TMP30]] = add nuw nsw i32 [[TMP11]], 1
; IPSCCP-NEXT: br label [[BB8]]
; IPSCCP: bb31:
; IPSCCP-NEXT: [[TMP32:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP9]], [[BB8]] ]
More information about the llvm-commits
mailing list