[llvm-branch-commits] [llvm-branch] r101103 - in /llvm/branches/Apple/Morbo: include/llvm/Analysis/ lib/Analysis/ lib/Transforms/Scalar/ test/CodeGen/X86/ test/Transforms/LoopStrengthReduce/
Dan Gohman
gohman at apple.com
Mon Apr 12 17:21:38 PDT 2010
Author: djg
Date: Mon Apr 12 19:21:38 2010
New Revision: 101103
URL: http://llvm.org/viewvc/llvm-project?rev=101103&view=rev
Log:
$ svn merge -c 100515 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100515 into '.':
U lib/Analysis/IVUsers.cpp
$ svn merge -c 100699 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100699 into '.':
U test/Transforms/LoopStrengthReduce/quadradic-exit-value.ll
A test/CodeGen/X86/multiple-loop-post-inc.ll
A include/llvm/Analysis/ScalarEvolutionNormalization.h
U include/llvm/Analysis/IVUsers.h
U include/llvm/Analysis/ScalarEvolutionExpander.h
G lib/Analysis/IVUsers.cpp
U lib/Analysis/ScalarEvolutionExpander.cpp
A lib/Analysis/ScalarEvolutionNormalization.cpp
U lib/Transforms/Scalar/IndVarSimplify.cpp
U lib/Transforms/Scalar/LoopStrengthReduce.cpp
$ svn merge -c 100748 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100748 into '.':
U test/CodeGen/X86/multiple-loop-post-inc.ll
G lib/Transforms/Scalar/LoopStrengthReduce.cpp
$ svn merge -c 100749 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100749 into '.':
U lib/Analysis/ScalarEvolution.cpp
$ svn merge -c 100828 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100828 into '.':
G lib/Transforms/Scalar/LoopStrengthReduce.cpp
$ svn merge -c 100839 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100839 into '.':
A test/Transforms/LoopStrengthReduce/insert-positions.ll
G lib/Analysis/IVUsers.cpp
$ svn merge -c 100845 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100845 into '.':
G lib/Transforms/Scalar/LoopStrengthReduce.cpp
$ svn merge -c 100874 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100874 into '.':
G lib/Analysis/ScalarEvolutionExpander.cpp
$ svn merge -c 100883 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100883 into '.':
G lib/Transforms/Scalar/LoopStrengthReduce.cpp
$ svn merge -c 100884 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100884 into '.':
A test/Transforms/LoopStrengthReduce/uglygep.ll
G lib/Analysis/ScalarEvolutionExpander.cpp
$ svn merge -c 100906 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r100906 into '.':
U test/Transforms/LoopStrengthReduce/uglygep.ll
G lib/Transforms/Scalar/LoopStrengthReduce.cpp
$ svn merge -c 101033 https://djg@llvm.org/svn/llvm-project/llvm/trunk
--- Merging r101033 into '.':
G lib/Transforms/Scalar/LoopStrengthReduce.cpp
Added:
llvm/branches/Apple/Morbo/include/llvm/Analysis/ScalarEvolutionNormalization.h
- copied unchanged from r100699, llvm/trunk/include/llvm/Analysis/ScalarEvolutionNormalization.h
llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolutionNormalization.cpp
- copied unchanged from r100699, llvm/trunk/lib/Analysis/ScalarEvolutionNormalization.cpp
llvm/branches/Apple/Morbo/test/CodeGen/X86/multiple-loop-post-inc.ll
- copied, changed from r100699, llvm/trunk/test/CodeGen/X86/multiple-loop-post-inc.ll
llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/insert-positions.ll
- copied unchanged from r100839, llvm/trunk/test/Transforms/LoopStrengthReduce/insert-positions.ll
llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/uglygep.ll
- copied, changed from r100884, llvm/trunk/test/Transforms/LoopStrengthReduce/uglygep.ll
Modified:
llvm/branches/Apple/Morbo/include/llvm/Analysis/IVUsers.h
llvm/branches/Apple/Morbo/include/llvm/Analysis/ScalarEvolutionExpander.h
llvm/branches/Apple/Morbo/lib/Analysis/IVUsers.cpp
llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolution.cpp
llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolutionExpander.cpp
llvm/branches/Apple/Morbo/lib/Transforms/Scalar/IndVarSimplify.cpp
llvm/branches/Apple/Morbo/lib/Transforms/Scalar/LoopStrengthReduce.cpp
llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/quadradic-exit-value.ll
Modified: llvm/branches/Apple/Morbo/include/llvm/Analysis/IVUsers.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/include/llvm/Analysis/IVUsers.h?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/include/llvm/Analysis/IVUsers.h (original)
+++ llvm/branches/Apple/Morbo/include/llvm/Analysis/IVUsers.h Mon Apr 12 19:21:38 2010
@@ -16,6 +16,7 @@
#define LLVM_ANALYSIS_IVUSERS_H
#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/ScalarEvolutionNormalization.h"
#include "llvm/Support/ValueHandle.h"
namespace llvm {
@@ -26,17 +27,18 @@
class IVUsers;
class ScalarEvolution;
class SCEV;
+class IVUsers;
/// IVStrideUse - Keep track of one use of a strided induction variable.
/// The Expr member keeps track of the expression, User is the actual user
/// instruction of the operand, and 'OperandValToReplace' is the operand of
/// the User that is the use.
class IVStrideUse : public CallbackVH, public ilist_node<IVStrideUse> {
+ friend class IVUsers;
public:
- IVStrideUse(IVUsers *P, const SCEV *S, const SCEV *Off,
+ IVStrideUse(IVUsers *P, const SCEV *E,
Instruction* U, Value *O)
- : CallbackVH(U), Parent(P), Stride(S), Offset(Off),
- OperandValToReplace(O), IsUseOfPostIncrementedValue(false) {
+ : CallbackVH(U), Parent(P), Expr(E), OperandValToReplace(O) {
}
/// getUser - Return the user instruction for this use.
@@ -53,24 +55,16 @@
/// this IVStrideUse.
IVUsers *getParent() const { return Parent; }
- /// getStride - Return the expression for the stride for the use.
- const SCEV *getStride() const { return Stride; }
+ /// getExpr - Return the expression for the use.
+ const SCEV *getExpr() const { return Expr; }
- /// setStride - Assign a new stride to this use.
- void setStride(const SCEV *Val) {
- Stride = Val;
- }
-
- /// getOffset - Return the offset to add to a theoretical induction
- /// variable that starts at zero and counts up by the stride to compute
- /// the value for the use. This always has the same type as the stride.
- const SCEV *getOffset() const { return Offset; }
-
- /// setOffset - Assign a new offset to this use.
- void setOffset(const SCEV *Val) {
- Offset = Val;
+ /// setExpr - Assign a new expression to this use.
+ void setExpr(const SCEV *Val) {
+ Expr = Val;
}
+ const SCEV *getStride(const Loop *L) const;
+
/// getOperandValToReplace - Return the Value of the operand in the user
/// instruction that this IVStrideUse is representing.
Value *getOperandValToReplace() const {
@@ -83,37 +77,30 @@
OperandValToReplace = Op;
}
- /// isUseOfPostIncrementedValue - True if this should use the
- /// post-incremented version of this IV, not the preincremented version.
- /// This can only be set in special cases, such as the terminating setcc
- /// instruction for a loop or uses dominated by the loop.
- bool isUseOfPostIncrementedValue() const {
- return IsUseOfPostIncrementedValue;
+ /// getPostIncLoops - Return the set of loops for which the expression has
+ /// been adjusted to use post-inc mode.
+ const PostIncLoopSet &getPostIncLoops() const {
+ return PostIncLoops;
}
- /// setIsUseOfPostIncrmentedValue - set the flag that indicates whether
- /// this is a post-increment use.
- void setIsUseOfPostIncrementedValue(bool Val) {
- IsUseOfPostIncrementedValue = Val;
- }
+ /// transformToPostInc - Transform the expression to post-inc form for the
+ /// given loop.
+ void transformToPostInc(const Loop *L);
private:
/// Parent - a pointer to the IVUsers that owns this IVStrideUse.
IVUsers *Parent;
- /// Stride - The stride for this use.
- const SCEV *Stride;
-
- /// Offset - The offset to add to the base induction expression.
- const SCEV *Offset;
+ /// Expr - The expression for this use.
+ const SCEV *Expr;
/// OperandValToReplace - The Value of the operand in the user instruction
/// that this IVStrideUse is representing.
WeakVH OperandValToReplace;
- /// IsUseOfPostIncrementedValue - True if this should use the
- /// post-incremented version of this IV, not the preincremented version.
- bool IsUseOfPostIncrementedValue;
+ /// PostIncLoops - The set of loops for which Expr has been adjusted to
+ /// use post-inc mode. This corresponds with SCEVExpander's post-inc concept.
+ PostIncLoopSet PostIncLoops;
/// Deleted - Implementation of CallbackVH virtual function to
/// receive notification when the User is deleted.
@@ -174,18 +161,13 @@
/// return true. Otherwise, return false.
bool AddUsersIfInteresting(Instruction *I);
- IVStrideUse &AddUser(const SCEV *Stride, const SCEV *Offset,
+ IVStrideUse &AddUser(const SCEV *Expr,
Instruction *User, Value *Operand);
/// getReplacementExpr - Return a SCEV expression which computes the
/// value of the OperandValToReplace of the given IVStrideUse.
const SCEV *getReplacementExpr(const IVStrideUse &U) const;
- /// getCanonicalExpr - Return a SCEV expression which computes the
- /// value of the SCEV of the given IVStrideUse, ignoring the
- /// isUseOfPostIncrementedValue flag.
- const SCEV *getCanonicalExpr(const IVStrideUse &U) const;
-
typedef ilist<IVStrideUse>::iterator iterator;
typedef ilist<IVStrideUse>::const_iterator const_iterator;
iterator begin() { return IVUses.begin(); }
Modified: llvm/branches/Apple/Morbo/include/llvm/Analysis/ScalarEvolutionExpander.h
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/include/llvm/Analysis/ScalarEvolutionExpander.h?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/include/llvm/Analysis/ScalarEvolutionExpander.h (original)
+++ llvm/branches/Apple/Morbo/include/llvm/Analysis/ScalarEvolutionExpander.h Mon Apr 12 19:21:38 2010
@@ -15,6 +15,7 @@
#define LLVM_ANALYSIS_SCALAREVOLUTION_EXPANDER_H
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ScalarEvolutionNormalization.h"
#include "llvm/Support/IRBuilder.h"
#include "llvm/Support/TargetFolder.h"
#include <set>
@@ -32,12 +33,12 @@
InsertedExpressions;
std::set<Value*> InsertedValues;
- /// PostIncLoop - When non-null, expanded addrecs referring to the given
- /// loop expanded in post-inc mode. For example, expanding {1,+,1}<L> in
- /// post-inc mode returns the add instruction that adds one to the phi
- /// for {0,+,1}<L>, as opposed to a new phi starting at 1. This is only
- /// supported in non-canonical mode.
- const Loop *PostIncLoop;
+ /// PostIncLoops - Addrecs referring to any of the given loops are expanded
+ /// in post-inc mode. For example, expanding {1,+,1}<L> in post-inc mode
+ /// returns the add instruction that adds one to the phi for {0,+,1}<L>,
+ /// as opposed to a new phi starting at 1. This is only supported in
+ /// non-canonical mode.
+ PostIncLoopSet PostIncLoops;
/// IVIncInsertPos - When this is non-null, addrecs expanded in the
/// loop it indicates should be inserted with increments at
@@ -62,7 +63,7 @@
public:
/// SCEVExpander - Construct a SCEVExpander in "canonical" mode.
explicit SCEVExpander(ScalarEvolution &se)
- : SE(se), PostIncLoop(0), IVIncInsertLoop(0), CanonicalMode(true),
+ : SE(se), IVIncInsertLoop(0), CanonicalMode(true),
Builder(se.getContext(), TargetFolder(se.TD)) {}
/// clear - Erase the contents of the InsertedExpressions map so that users
@@ -89,14 +90,18 @@
IVIncInsertPos = Pos;
}
- /// setPostInc - If L is non-null, enable post-inc expansion for addrecs
- /// referring to the given loop. If L is null, disable post-inc expansion
- /// completely. Post-inc expansion is only supported in non-canonical
+ /// setPostInc - Enable post-inc expansion for addrecs referring to the
+ /// given loops. Post-inc expansion is only supported in non-canonical
/// mode.
- void setPostInc(const Loop *L) {
+ void setPostInc(const PostIncLoopSet &L) {
assert(!CanonicalMode &&
"Post-inc expansion is not supported in CanonicalMode");
- PostIncLoop = L;
+ PostIncLoops = L;
+ }
+
+ /// clearPostInc - Disable all post-inc expansion.
+ void clearPostInc() {
+ PostIncLoops.clear();
}
/// disableCanonicalMode - Disable the behavior of expanding expressions in
Modified: llvm/branches/Apple/Morbo/lib/Analysis/IVUsers.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/lib/Analysis/IVUsers.cpp?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/lib/Analysis/IVUsers.cpp (original)
+++ llvm/branches/Apple/Morbo/lib/Analysis/IVUsers.cpp Mon Apr 12 19:21:38 2010
@@ -62,120 +62,34 @@
Ops.push_back(S);
}
-/// getSCEVStartAndStride - Compute the start and stride of this expression,
-/// returning false if the expression is not a start/stride pair, or true if it
-/// is. The stride must be a loop invariant expression, but the start may be
-/// a mix of loop invariant and loop variant expressions. The start cannot,
-/// however, contain an AddRec from a different loop, unless that loop is an
-/// outer loop of the current loop.
-static bool getSCEVStartAndStride(const SCEV *&SH, Loop *L, Loop *UseLoop,
- const SCEV *&Start, const SCEV *&Stride,
- ScalarEvolution *SE, DominatorTree *DT) {
- const SCEV *TheAddRec = Start; // Initialize to zero.
-
- // If the outer level is an AddExpr, the operands are all start values except
- // for a nested AddRecExpr.
- if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(SH)) {
- for (unsigned i = 0, e = AE->getNumOperands(); i != e; ++i)
- if (const SCEVAddRecExpr *AddRec =
- dyn_cast<SCEVAddRecExpr>(AE->getOperand(i)))
- TheAddRec = SE->getAddExpr(AddRec, TheAddRec);
- else
- Start = SE->getAddExpr(Start, AE->getOperand(i));
- } else if (isa<SCEVAddRecExpr>(SH)) {
- TheAddRec = SH;
- } else {
- return false; // not analyzable.
- }
-
- // Break down TheAddRec into its component parts.
- SmallVector<const SCEV *, 4> Subexprs;
- CollectSubexprs(TheAddRec, Subexprs, *SE);
-
- // Look for an addrec on the current loop among the parts.
- const SCEV *AddRecStride = 0;
- for (SmallVectorImpl<const SCEV *>::iterator I = Subexprs.begin(),
- E = Subexprs.end(); I != E; ++I) {
- const SCEV *S = *I;
- if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
- if (AR->getLoop() == L) {
- *I = AR->getStart();
- AddRecStride = AR->getStepRecurrence(*SE);
- break;
- }
- }
- if (!AddRecStride)
- return false;
+/// isInteresting - Test whether the given expression is "interesting" when
+/// used by the given expression, within the context of analyzing the
+/// given loop.
+static bool isInteresting(const SCEV *S, const Instruction *I, const Loop *L) {
+ // Anything loop-invariant is interesting.
+ if (!isa<SCEVUnknown>(S) && S->isLoopInvariant(L))
+ return true;
- // Add up everything else into a start value (which may not be
- // loop-invariant).
- const SCEV *AddRecStart = SE->getAddExpr(Subexprs);
-
- // Use getSCEVAtScope to attempt to simplify other loops out of
- // the picture.
- AddRecStart = SE->getSCEVAtScope(AddRecStart, UseLoop);
-
- Start = SE->getAddExpr(Start, AddRecStart);
-
- // If stride is an instruction, make sure it properly dominates the header.
- // Otherwise we could end up with a use before def situation.
- if (!isa<SCEVConstant>(AddRecStride)) {
- BasicBlock *Header = L->getHeader();
- if (!AddRecStride->properlyDominates(Header, DT))
- return false;
-
- DEBUG(dbgs() << "[";
- WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false);
- dbgs() << "] Variable stride: " << *AddRecStride << "\n");
+ // An addrec is interesting if it's affine or if it has an interesting start.
+ if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
+ // Keep things simple. Don't touch loop-variant strides.
+ if (AR->getLoop() == L)
+ return AR->isAffine() || !L->contains(I);
+ // Otherwise recurse to see if the start value is interesting.
+ return isInteresting(AR->getStart(), I, L);
}
- Stride = AddRecStride;
- return true;
-}
-
-/// IVUseShouldUsePostIncValue - We have discovered a "User" of an IV expression
-/// and now we need to decide whether the user should use the preinc or post-inc
-/// value. If this user should use the post-inc version of the IV, return true.
-///
-/// Choosing wrong here can break dominance properties (if we choose to use the
-/// post-inc value when we cannot) or it can end up adding extra live-ranges to
-/// the loop, resulting in reg-reg copies (if we use the pre-inc value when we
-/// should use the post-inc value).
-static bool IVUseShouldUsePostIncValue(Instruction *User, Instruction *IV,
- Loop *L, DominatorTree *DT) {
- // If the user is in the loop, use the preinc value.
- if (L->contains(User)) return false;
-
- BasicBlock *LatchBlock = L->getLoopLatch();
- if (!LatchBlock)
+ // An add is interesting if any of its operands is.
+ if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
+ for (SCEVAddExpr::op_iterator OI = Add->op_begin(), OE = Add->op_end();
+ OI != OE; ++OI)
+ if (isInteresting(*OI, I, L))
+ return true;
return false;
+ }
- // Ok, the user is outside of the loop. If it is dominated by the latch
- // block, use the post-inc value.
- if (DT->dominates(LatchBlock, User->getParent()))
- return true;
-
- // There is one case we have to be careful of: PHI nodes. These little guys
- // can live in blocks that are not dominated by the latch block, but (since
- // their uses occur in the predecessor block, not the block the PHI lives in)
- // should still use the post-inc value. Check for this case now.
- PHINode *PN = dyn_cast<PHINode>(User);
- if (!PN) return false; // not a phi, not dominated by latch block.
-
- // Look at all of the uses of IV by the PHI node. If any use corresponds to
- // a block that is not dominated by the latch block, give up and use the
- // preincremented value.
- unsigned NumUses = 0;
- for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
- if (PN->getIncomingValue(i) == IV) {
- ++NumUses;
- if (!DT->dominates(LatchBlock, PN->getIncomingBlock(i)))
- return false;
- }
-
- // Okay, all uses of IV by PN are in predecessor blocks that really are
- // dominated by the latch block. Use the post-incremented value.
- return true;
+ // Nothing else is interesting here.
+ return false;
}
/// AddUsersIfInteresting - Inspect the specified instruction. If it is a
@@ -196,16 +110,9 @@
const SCEV *ISE = SE->getSCEV(I);
if (isa<SCEVCouldNotCompute>(ISE)) return false;
- // Get the start and stride for this expression.
- Loop *UseLoop = LI->getLoopFor(I->getParent());
- const SCEV *Start = SE->getIntegerSCEV(0, ISE->getType());
- const SCEV *Stride = Start;
-
- if (!getSCEVStartAndStride(ISE, L, UseLoop, Start, Stride, SE, DT))
- return false; // Non-reducible symbolic expression, bail out.
-
- // Keep things simple. Don't touch loop-variant strides.
- if (!Stride->isLoopInvariant(L) && L->contains(I))
+ // If we've come to an uninteresting expression, stop the traversal and
+ // call this a user.
+ if (!isInteresting(ISE, I, L))
return false;
SmallPtrSet<Instruction *, 4> UniqueUsers;
@@ -241,27 +148,24 @@
}
if (AddUserToIVUsers) {
- // Okay, we found a user that we cannot reduce. Analyze the instruction
- // and decide what to do with it. If we are a use inside of the loop, use
- // the value before incrementation, otherwise use it after incrementation.
- if (IVUseShouldUsePostIncValue(User, I, L, DT)) {
- // The value used will be incremented by the stride more than we are
- // expecting, so subtract this off.
- const SCEV *NewStart = SE->getMinusSCEV(Start, Stride);
- IVUses.push_back(new IVStrideUse(this, Stride, NewStart, User, I));
- IVUses.back().setIsUseOfPostIncrementedValue(true);
- DEBUG(dbgs() << " USING POSTINC SCEV, START=" << *NewStart<< "\n");
- } else {
- IVUses.push_back(new IVStrideUse(this, Stride, Start, User, I));
- }
+ // Okay, we found a user that we cannot reduce.
+ IVUses.push_back(new IVStrideUse(this, ISE, User, I));
+ IVStrideUse &NewUse = IVUses.back();
+ // Transform the expression into a normalized form.
+ NewUse.Expr =
+ TransformForPostIncUse(NormalizeAutodetect, NewUse.Expr,
+ User, I,
+ NewUse.PostIncLoops,
+ *SE, *DT);
+ DEBUG(dbgs() << " NORMALIZED TO: " << *NewUse.Expr << '\n');
}
}
return true;
}
-IVStrideUse &IVUsers::AddUser(const SCEV *Stride, const SCEV *Offset,
+IVStrideUse &IVUsers::AddUser(const SCEV *Expr,
Instruction *User, Value *Operand) {
- IVUses.push_back(new IVStrideUse(this, Stride, Offset, User, Operand));
+ IVUses.push_back(new IVStrideUse(this, Expr, User, Operand));
return IVUses.back();
}
@@ -295,30 +199,10 @@
/// getReplacementExpr - Return a SCEV expression which computes the
/// value of the OperandValToReplace of the given IVStrideUse.
const SCEV *IVUsers::getReplacementExpr(const IVStrideUse &U) const {
- // Start with zero.
- const SCEV *RetVal = SE->getIntegerSCEV(0, U.getStride()->getType());
- // Create the basic add recurrence.
- RetVal = SE->getAddRecExpr(RetVal, U.getStride(), L);
- // Add the offset in a separate step, because it may be loop-variant.
- RetVal = SE->getAddExpr(RetVal, U.getOffset());
- // For uses of post-incremented values, add an extra stride to compute
- // the actual replacement value.
- if (U.isUseOfPostIncrementedValue())
- RetVal = SE->getAddExpr(RetVal, U.getStride());
- return RetVal;
-}
-
-/// getCanonicalExpr - Return a SCEV expression which computes the
-/// value of the SCEV of the given IVStrideUse, ignoring the
-/// isUseOfPostIncrementedValue flag.
-const SCEV *IVUsers::getCanonicalExpr(const IVStrideUse &U) const {
- // Start with zero.
- const SCEV *RetVal = SE->getIntegerSCEV(0, U.getStride()->getType());
- // Create the basic add recurrence.
- RetVal = SE->getAddRecExpr(RetVal, U.getStride(), L);
- // Add the offset in a separate step, because it may be loop-variant.
- RetVal = SE->getAddExpr(RetVal, U.getOffset());
- return RetVal;
+ PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(U.PostIncLoops);
+ return TransformForPostIncUse(Denormalize, U.getExpr(),
+ U.getUser(), U.getOperandValToReplace(),
+ Loops, *SE, *DT);
}
void IVUsers::print(raw_ostream &OS, const Module *M) const {
@@ -339,8 +223,13 @@
WriteAsOperand(OS, UI->getOperandValToReplace(), false);
OS << " = "
<< *getReplacementExpr(*UI);
- if (UI->isUseOfPostIncrementedValue())
- OS << " (post-inc)";
+ for (PostIncLoopSet::const_iterator
+ I = UI->PostIncLoops.begin(),
+ E = UI->PostIncLoops.end(); I != E; ++I) {
+ OS << " (post-inc with loop ";
+ WriteAsOperand(OS, (*I)->getHeader(), false);
+ OS << ")";
+ }
OS << " in ";
UI->getUser()->print(OS, &Annotator);
OS << '\n';
@@ -356,6 +245,39 @@
IVUses.clear();
}
+static const SCEVAddRecExpr *findAddRecForLoop(const SCEV *S, const Loop *L) {
+ if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
+ if (AR->getLoop() == L)
+ return AR;
+ return findAddRecForLoop(AR->getStart(), L);
+ }
+
+ if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
+ for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
+ I != E; ++I)
+ if (const SCEVAddRecExpr *AR = findAddRecForLoop(*I, L))
+ return AR;
+ return 0;
+ }
+
+ return 0;
+}
+
+const SCEV *IVStrideUse::getStride(const Loop *L) const {
+ if (const SCEVAddRecExpr *AR = findAddRecForLoop(getExpr(), L))
+ return AR->getStepRecurrence(*Parent->SE);
+ return 0;
+}
+
+void IVStrideUse::transformToPostInc(const Loop *L) {
+ PostIncLoopSet Loops;
+ Loops.insert(L);
+ Expr = TransformForPostIncUse(Normalize, Expr,
+ getUser(), getOperandValToReplace(),
+ Loops, *Parent->SE, *Parent->DT);
+ PostIncLoops.insert(L);
+}
+
void IVStrideUse::deleted() {
// Remove this user from the list.
Parent->IVUses.erase(this);
Modified: llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolution.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolution.cpp?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolution.cpp (original)
+++ llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolution.cpp Mon Apr 12 19:21:38 2010
@@ -3123,8 +3123,6 @@
return getConstant(CI);
else if (isa<ConstantPointerNull>(V))
return getIntegerSCEV(0, V->getType());
- else if (isa<UndefValue>(V))
- return getIntegerSCEV(0, V->getType());
else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee());
else
Modified: llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolutionExpander.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolutionExpander.cpp?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolutionExpander.cpp (original)
+++ llvm/branches/Apple/Morbo/lib/Analysis/ScalarEvolutionExpander.cpp Mon Apr 12 19:21:38 2010
@@ -705,9 +705,11 @@
Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, Sum);
} else if (const PointerType *PTy = dyn_cast<PointerType>(Op->getType())) {
// The running sum is an integer, and there's a pointer at this level.
- // Try to form a getelementptr.
+ // Try to form a getelementptr. If the running sum is instructions,
+ // use a SCEVUnknown to avoid re-analyzing them.
SmallVector<const SCEV *, 4> NewOps;
- NewOps.push_back(SE.getUnknown(Sum));
+ NewOps.push_back(isa<Instruction>(Sum) ? SE.getUnknown(Sum) :
+ SE.getSCEV(Sum));
for (++I; I != E && I->first == CurLoop; ++I)
NewOps.push_back(I->second);
Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op));
@@ -966,9 +968,12 @@
// Determine a normalized form of this expression, which is the expression
// before any post-inc adjustment is made.
const SCEVAddRecExpr *Normalized = S;
- if (L == PostIncLoop) {
- const SCEV *Step = S->getStepRecurrence(SE);
- Normalized = cast<SCEVAddRecExpr>(SE.getMinusSCEV(S, Step));
+ if (PostIncLoops.count(L)) {
+ PostIncLoopSet Loops;
+ Loops.insert(L);
+ Normalized =
+ cast<SCEVAddRecExpr>(TransformForPostIncUse(Normalize, S, 0, 0,
+ Loops, SE, *SE.DT));
}
// Strip off any non-loop-dominating component from the addrec start.
@@ -1002,7 +1007,7 @@
// Accommodate post-inc mode, if necessary.
Value *Result;
- if (L != PostIncLoop)
+ if (!PostIncLoops.count(L))
Result = PN;
else {
// In PostInc mode, use the post-incremented value.
@@ -1274,7 +1279,7 @@
// If the SCEV is computable at this level, insert it into the header
// after the PHIs (and after any other instructions that we've inserted
// there) so that it is guaranteed to dominate any user inside the loop.
- if (L && S->hasComputableLoopEvolution(L) && L != PostIncLoop)
+ if (L && S->hasComputableLoopEvolution(L) && !PostIncLoops.count(L))
InsertPt = L->getHeader()->getFirstNonPHI();
while (isInsertedInstruction(InsertPt) || isa<DbgInfoIntrinsic>(InsertPt))
InsertPt = llvm::next(BasicBlock::iterator(InsertPt));
@@ -1296,7 +1301,7 @@
Value *V = visit(S);
// Remember the expanded value for this SCEV at this location.
- if (!PostIncLoop)
+ if (PostIncLoops.empty())
InsertedExpressions[std::make_pair(S, InsertPt)] = V;
restoreInsertPoint(SaveInsertBB, SaveInsertPt);
@@ -1304,7 +1309,7 @@
}
void SCEVExpander::rememberInstruction(Value *I) {
- if (!PostIncLoop)
+ if (PostIncLoops.empty())
InsertedValues.insert(I);
// If we just claimed an existing instruction and that instruction had
Modified: llvm/branches/Apple/Morbo/lib/Transforms/Scalar/IndVarSimplify.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/lib/Transforms/Scalar/IndVarSimplify.cpp?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/lib/Transforms/Scalar/IndVarSimplify.cpp (original)
+++ llvm/branches/Apple/Morbo/lib/Transforms/Scalar/IndVarSimplify.cpp Mon Apr 12 19:21:38 2010
@@ -450,6 +450,46 @@
return Changed;
}
+// FIXME: It is an extremely bad idea to indvar substitute anything more
+// complex than affine induction variables. Doing so will put expensive
+// polynomial evaluations inside of the loop, and the str reduction pass
+// currently can only reduce affine polynomials. For now just disable
+// indvar subst on anything more complex than an affine addrec, unless
+// it can be expanded to a trivial value.
+static bool isSafe(const SCEV *S, const Loop *L) {
+ // Loop-invariant values are safe.
+ if (S->isLoopInvariant(L)) return true;
+
+ // Affine addrecs are safe. Non-affine are not, because LSR doesn't know how
+ // to transform them into efficient code.
+ if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
+ return AR->isAffine();
+
+ // An add is safe it all its operands are safe.
+ if (const SCEVCommutativeExpr *Commutative = dyn_cast<SCEVCommutativeExpr>(S)) {
+ for (SCEVCommutativeExpr::op_iterator I = Commutative->op_begin(),
+ E = Commutative->op_end(); I != E; ++I)
+ if (!isSafe(*I, L)) return false;
+ return true;
+ }
+
+ // A cast is safe if its operand is.
+ if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S))
+ return isSafe(C->getOperand(), L);
+
+ // A udiv is safe if its operands are.
+ if (const SCEVUDivExpr *UD = dyn_cast<SCEVUDivExpr>(S))
+ return isSafe(UD->getLHS(), L) &&
+ isSafe(UD->getRHS(), L);
+
+ // SCEVUnknown is always safe.
+ if (isa<SCEVUnknown>(S))
+ return true;
+
+ // Nothing else is safe.
+ return false;
+}
+
void IndVarSimplify::RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter) {
SmallVector<WeakVH, 16> DeadInsts;
@@ -461,7 +501,6 @@
// the need for the code evaluation methods to insert induction variables
// of different sizes.
for (IVUsers::iterator UI = IU->begin(), E = IU->end(); UI != E; ++UI) {
- const SCEV *Stride = UI->getStride();
Value *Op = UI->getOperandValToReplace();
const Type *UseTy = Op->getType();
Instruction *User = UI->getUser();
@@ -482,7 +521,7 @@
// currently can only reduce affine polynomials. For now just disable
// indvar subst on anything more complex than an affine addrec, unless
// it can be expanded to a trivial value.
- if (!AR->isLoopInvariant(L) && !Stride->isLoopInvariant(L))
+ if (!isSafe(AR, L))
continue;
// Determine the insertion point for this user. By default, insert
Modified: llvm/branches/Apple/Morbo/lib/Transforms/Scalar/LoopStrengthReduce.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/lib/Transforms/Scalar/LoopStrengthReduce.cpp?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/lib/Transforms/Scalar/LoopStrengthReduce.cpp (original)
+++ llvm/branches/Apple/Morbo/lib/Transforms/Scalar/LoopStrengthReduce.cpp Mon Apr 12 19:21:38 2010
@@ -262,11 +262,15 @@
SmallVector<const SCEV *, 4> Bad;
DoInitialMatch(S, L, Good, Bad, SE, DT);
if (!Good.empty()) {
- BaseRegs.push_back(SE.getAddExpr(Good));
+ const SCEV *Sum = SE.getAddExpr(Good);
+ if (!Sum->isZero())
+ BaseRegs.push_back(Sum);
AM.HasBaseReg = true;
}
if (!Bad.empty()) {
- BaseRegs.push_back(SE.getAddExpr(Bad));
+ const SCEV *Sum = SE.getAddExpr(Bad);
+ if (!Sum->isZero())
+ BaseRegs.push_back(Sum);
AM.HasBaseReg = true;
}
}
@@ -781,10 +785,10 @@
/// will be replaced.
Value *OperandValToReplace;
- /// PostIncLoop - If this user is to use the post-incremented value of an
+ /// PostIncLoops - If this user is to use the post-incremented value of an
/// induction variable, this variable is non-null and holds the loop
/// associated with the induction variable.
- const Loop *PostIncLoop;
+ PostIncLoopSet PostIncLoops;
/// LUIdx - The index of the LSRUse describing the expression which
/// this fixup needs, minus an offset (below).
@@ -795,6 +799,8 @@
/// offsets, for example in an unrolled loop.
int64_t Offset;
+ bool isUseFullyOutsideLoop(const Loop *L) const;
+
LSRFixup();
void print(raw_ostream &OS) const;
@@ -804,9 +810,24 @@
}
LSRFixup::LSRFixup()
- : UserInst(0), OperandValToReplace(0), PostIncLoop(0),
+ : UserInst(0), OperandValToReplace(0),
LUIdx(~size_t(0)), Offset(0) {}
+/// isUseFullyOutsideLoop - Test whether this fixup always uses its
+/// value outside of the given loop.
+bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const {
+ // PHI nodes use their value in their incoming blocks.
+ if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) {
+ for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
+ if (PN->getIncomingValue(i) == OperandValToReplace &&
+ L->contains(PN->getIncomingBlock(i)))
+ return false;
+ return true;
+ }
+
+ return !L->contains(UserInst);
+}
+
void LSRFixup::print(raw_ostream &OS) const {
OS << "UserInst=";
// Store is common and interesting enough to be worth special-casing.
@@ -821,9 +842,10 @@
OS << ", OperandValToReplace=";
WriteAsOperand(OS, OperandValToReplace, /*PrintType=*/false);
- if (PostIncLoop) {
+ for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(),
+ E = PostIncLoops.end(); I != E; ++I) {
OS << ", PostIncLoop=";
- WriteAsOperand(OS, PostIncLoop->getHeader(), /*PrintType=*/false);
+ WriteAsOperand(OS, (*I)->getHeader(), /*PrintType=*/false);
}
if (LUIdx != ~size_t(0))
@@ -1135,6 +1157,7 @@
IVUsers &IU;
ScalarEvolution &SE;
DominatorTree &DT;
+ LoopInfo &LI;
const TargetLowering *const TLI;
Loop *const L;
bool Changed;
@@ -1214,6 +1237,13 @@
DenseSet<const SCEV *> &VisitedRegs) const;
void Solve(SmallVectorImpl<const Formula *> &Solution) const;
+ BasicBlock::iterator
+ HoistInsertPosition(BasicBlock::iterator IP,
+ const SmallVectorImpl<Instruction *> &Inputs) const;
+ BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP,
+ const LSRFixup &LF,
+ const LSRUse &LU) const;
+
Value *Expand(const LSRFixup &LF,
const Formula &F,
BasicBlock::iterator IP,
@@ -1545,8 +1575,9 @@
!DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) {
// Conservatively assume there may be reuse if the quotient of their
// strides could be a legal scale.
- const SCEV *A = CondUse->getStride();
- const SCEV *B = UI->getStride();
+ const SCEV *A = CondUse->getStride(L);
+ const SCEV *B = UI->getStride(L);
+ if (!A || !B) continue;
if (SE.getTypeSizeInBits(A->getType()) !=
SE.getTypeSizeInBits(B->getType())) {
if (SE.getTypeSizeInBits(A->getType()) >
@@ -1598,7 +1629,7 @@
ExitingBlock->getInstList().insert(TermBr, Cond);
// Clone the IVUse, as the old use still exists!
- CondUse = &IU.AddUser(CondUse->getStride(), CondUse->getOffset(),
+ CondUse = &IU.AddUser(CondUse->getExpr(),
Cond, CondUse->getOperandValToReplace());
TermBr->replaceUsesOfWith(OldCond, Cond);
}
@@ -1607,9 +1638,7 @@
// If we get to here, we know that we can transform the setcc instruction to
// use the post-incremented version of the IV, allowing us to coalesce the
// live ranges for the IV correctly.
- CondUse->setOffset(SE.getMinusSCEV(CondUse->getOffset(),
- CondUse->getStride()));
- CondUse->setIsUseOfPostIncrementedValue(true);
+ CondUse->transformToPostInc(L);
Changed = true;
PostIncs.insert(Cond);
@@ -1717,19 +1746,24 @@
SmallSetVector<const SCEV *, 4> Strides;
// Collect interesting types and strides.
+ SmallVector<const SCEV *, 4> Worklist;
for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
- const SCEV *Stride = UI->getStride();
+ const SCEV *Expr = UI->getExpr();
// Collect interesting types.
- Types.insert(SE.getEffectiveSCEVType(Stride->getType()));
-
- // Add the stride for this loop.
- Strides.insert(Stride);
+ Types.insert(SE.getEffectiveSCEVType(Expr->getType()));
- // Add strides for other mentioned loops.
- for (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(UI->getOffset());
- AR; AR = dyn_cast<SCEVAddRecExpr>(AR->getStart()))
- Strides.insert(AR->getStepRecurrence(SE));
+ // Add strides for mentioned loops.
+ Worklist.push_back(Expr);
+ do {
+ const SCEV *S = Worklist.pop_back_val();
+ if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
+ Strides.insert(AR->getStepRecurrence(SE));
+ Worklist.push_back(AR->getStart());
+ } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
+ Worklist.insert(Worklist.end(), Add->op_begin(), Add->op_end());
+ }
+ } while (!Worklist.empty());
}
// Compute interesting factors from the set of interesting strides.
@@ -1776,8 +1810,7 @@
LSRFixup &LF = getNewFixup();
LF.UserInst = UI->getUser();
LF.OperandValToReplace = UI->getOperandValToReplace();
- if (UI->isUseOfPostIncrementedValue())
- LF.PostIncLoop = L;
+ LF.PostIncLoops = UI->getPostIncLoops();
LSRUse::KindType Kind = LSRUse::Basic;
const Type *AccessTy = 0;
@@ -1786,7 +1819,7 @@
AccessTy = getAccessType(LF.UserInst);
}
- const SCEV *S = IU.getCanonicalExpr(*UI);
+ const SCEV *S = UI->getExpr();
// Equality (== and !=) ICmps are special. We can rewrite (i == N) as
// (N - i == 0), and this allows (N - i) to be the expression that we work
@@ -1824,7 +1857,7 @@
LF.LUIdx = P.first;
LF.Offset = P.second;
LSRUse &LU = Uses[LF.LUIdx];
- LU.AllFixupsOutsideLoop &= !L->contains(LF.UserInst);
+ LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
// If this is the first use of this LSRUse, give it a formula.
if (LU.Formulae.empty()) {
@@ -1918,9 +1951,17 @@
continue;
// Ignore uses which are part of other SCEV expressions, to avoid
// analyzing them multiple times.
- if (SE.isSCEVable(UserInst->getType()) &&
- !isa<SCEVUnknown>(SE.getSCEV(const_cast<Instruction *>(UserInst))))
- continue;
+ if (SE.isSCEVable(UserInst->getType())) {
+ const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst));
+ // If the user is a no-op, look through to its uses.
+ if (!isa<SCEVUnknown>(UserS))
+ continue;
+ if (UserS == U) {
+ Worklist.push_back(
+ SE.getUnknown(const_cast<Instruction *>(UserInst)));
+ continue;
+ }
+ }
// Ignore icmp instructions which are already being analyzed.
if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) {
unsigned OtherIdx = !UI.getOperandNo();
@@ -1936,7 +1977,7 @@
LF.LUIdx = P.first;
LF.Offset = P.second;
LSRUse &LU = Uses[LF.LUIdx];
- LU.AllFixupsOutsideLoop &= L->contains(LF.UserInst);
+ LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
InsertSupplementalFormula(U, LU, LF.LUIdx);
CountRegisters(LU.Formulae.back(), Uses.size() - 1);
break;
@@ -2776,37 +2817,33 @@
return Node->getBlock();
}
-Value *LSRInstance::Expand(const LSRFixup &LF,
- const Formula &F,
- BasicBlock::iterator IP,
- SCEVExpander &Rewriter,
- SmallVectorImpl<WeakVH> &DeadInsts) const {
- const LSRUse &LU = Uses[LF.LUIdx];
+/// HoistInsertPosition - Helper for AdjustInsertPositionForExpand. Climb up
+/// the dominator tree far as we can go while still being dominated by the
+/// input positions. This helps canonicalize the insert position, which
+/// encourages sharing.
+BasicBlock::iterator
+LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
+ const SmallVectorImpl<Instruction *> &Inputs)
+ const {
+ for (;;) {
+ const Loop *IPLoop = LI.getLoopFor(IP->getParent());
+ unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0;
- // Then, collect some instructions which we will remain dominated by when
- // expanding the replacement. These must be dominated by any operands that
- // will be required in the expansion.
- SmallVector<Instruction *, 4> Inputs;
- if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace))
- Inputs.push_back(I);
- if (LU.Kind == LSRUse::ICmpZero)
- if (Instruction *I =
- dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1)))
- Inputs.push_back(I);
- if (LF.PostIncLoop) {
- if (!L->contains(LF.UserInst))
- Inputs.push_back(L->getLoopLatch()->getTerminator());
- else
- Inputs.push_back(IVIncInsertPos);
- }
+ BasicBlock *IDom;
+ for (BasicBlock *Rung = IP->getParent(); ; Rung = IDom) {
+ IDom = getImmediateDominator(Rung, DT);
+ if (!IDom) return IP;
+
+ // Don't climb into a loop though.
+ const Loop *IDomLoop = LI.getLoopFor(IDom);
+ unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0;
+ if (IDomDepth <= IPLoopDepth &&
+ (IDomDepth != IPLoopDepth || IDomLoop == IPLoop))
+ break;
+ }
- // Then, climb up the immediate dominator tree as far as we can go while
- // still being dominated by the input positions.
- for (;;) {
bool AllDominate = true;
Instruction *BetterPos = 0;
- BasicBlock *IDom = getImmediateDominator(IP->getParent(), DT);
- if (!IDom) break;
Instruction *Tentative = IDom->getTerminator();
for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(),
E = Inputs.end(); I != E; ++I) {
@@ -2815,6 +2852,8 @@
AllDominate = false;
break;
}
+ // Attempt to find an insert position in the middle of the block,
+ // instead of at the end, so that it can be used for other expansions.
if (IDom == Inst->getParent() &&
(!BetterPos || DT.dominates(BetterPos, Inst)))
BetterPos = next(BasicBlock::iterator(Inst));
@@ -2826,12 +2865,77 @@
else
IP = Tentative;
}
+
+ return IP;
+}
+
+/// AdjustInsertPositionForExpand - Determine an input position which will be
+/// dominated by the operands and which will dominate the result.
+BasicBlock::iterator
+LSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator IP,
+ const LSRFixup &LF,
+ const LSRUse &LU) const {
+ // Collect some instructions which must be dominated by the
+ // expanding replacement. These must be dominated by any operands that
+ // will be required in the expansion.
+ SmallVector<Instruction *, 4> Inputs;
+ if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace))
+ Inputs.push_back(I);
+ if (LU.Kind == LSRUse::ICmpZero)
+ if (Instruction *I =
+ dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1)))
+ Inputs.push_back(I);
+ if (LF.PostIncLoops.count(L)) {
+ if (LF.isUseFullyOutsideLoop(L))
+ Inputs.push_back(L->getLoopLatch()->getTerminator());
+ else
+ Inputs.push_back(IVIncInsertPos);
+ }
+ // The expansion must also be dominated by the increment positions of any
+ // loops it for which it is using post-inc mode.
+ for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(),
+ E = LF.PostIncLoops.end(); I != E; ++I) {
+ const Loop *PIL = *I;
+ if (PIL == L) continue;
+
+ // Be dominated by the loop exit.
+ SmallVector<BasicBlock *, 4> ExitingBlocks;
+ PIL->getExitingBlocks(ExitingBlocks);
+ if (!ExitingBlocks.empty()) {
+ BasicBlock *BB = ExitingBlocks[0];
+ for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i)
+ BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]);
+ Inputs.push_back(BB->getTerminator());
+ }
+ }
+
+ // Then, climb up the immediate dominator tree as far as we can go while
+ // still being dominated by the input positions.
+ IP = HoistInsertPosition(IP, Inputs);
+
+ // Don't insert instructions before PHI nodes.
while (isa<PHINode>(IP)) ++IP;
+
+ // Ignore debug intrinsics.
while (isa<DbgInfoIntrinsic>(IP)) ++IP;
+ return IP;
+}
+
+Value *LSRInstance::Expand(const LSRFixup &LF,
+ const Formula &F,
+ BasicBlock::iterator IP,
+ SCEVExpander &Rewriter,
+ SmallVectorImpl<WeakVH> &DeadInsts) const {
+ const LSRUse &LU = Uses[LF.LUIdx];
+
+ // Determine an input position which will be dominated by the operands and
+ // which will dominate the result.
+ IP = AdjustInsertPositionForExpand(IP, LF, LU);
+
// Inform the Rewriter if we have a post-increment use, so that it can
// perform an advantageous expansion.
- Rewriter.setPostInc(LF.PostIncLoop);
+ Rewriter.setPostInc(LF.PostIncLoops);
// This is the type that the user actually needs.
const Type *OpTy = LF.OperandValToReplace->getType();
@@ -2855,24 +2959,11 @@
const SCEV *Reg = *I;
assert(!Reg->isZero() && "Zero allocated in a base register!");
- // If we're expanding for a post-inc user for the add-rec's loop, make the
- // post-inc adjustment.
- const SCEV *Start = Reg;
- while (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Start)) {
- if (AR->getLoop() == LF.PostIncLoop) {
- Reg = SE.getAddExpr(Reg, AR->getStepRecurrence(SE));
- // If the user is inside the loop, insert the code after the increment
- // so that it is dominated by its operand. If the original insert point
- // was already dominated by the increment, keep it, because there may
- // be loop-variant operands that need to be respected also.
- if (L->contains(LF.UserInst) && !DT.dominates(IVIncInsertPos, IP)) {
- IP = IVIncInsertPos;
- while (isa<DbgInfoIntrinsic>(IP)) ++IP;
- }
- break;
- }
- Start = AR->getStart();
- }
+ // If we're expanding for a post-inc user, make the post-inc adjustment.
+ PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
+ Reg = TransformForPostIncUse(Denormalize, Reg,
+ LF.UserInst, LF.OperandValToReplace,
+ Loops, SE, DT);
Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP)));
}
@@ -2889,11 +2980,11 @@
if (F.AM.Scale != 0) {
const SCEV *ScaledS = F.ScaledReg;
- // If we're expanding for a post-inc user for the add-rec's loop, make the
- // post-inc adjustment.
- if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ScaledS))
- if (AR->getLoop() == LF.PostIncLoop)
- ScaledS = SE.getAddExpr(ScaledS, AR->getStepRecurrence(SE));
+ // If we're expanding for a post-inc user, make the post-inc adjustment.
+ PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
+ ScaledS = TransformForPostIncUse(Denormalize, ScaledS,
+ LF.UserInst, LF.OperandValToReplace,
+ Loops, SE, DT);
if (LU.Kind == LSRUse::ICmpZero) {
// An interesting way of "folding" with an icmp is to use a negated
@@ -2954,7 +3045,7 @@
Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP);
// We're done expanding now, so reset the rewriter.
- Rewriter.setPostInc(0);
+ Rewriter.clearPostInc();
// An ICmpZero Formula represents an ICmp which we're handling as a
// comparison against zero. Now that we've expanded an expression for that
@@ -3118,6 +3209,7 @@
: IU(P->getAnalysis<IVUsers>()),
SE(P->getAnalysis<ScalarEvolution>()),
DT(P->getAnalysis<DominatorTree>()),
+ LI(P->getAnalysis<LoopInfo>()),
TLI(tli), L(l), Changed(false), IVIncInsertPos(0) {
// If LoopSimplify form is not available, stay out of trouble.
@@ -3274,9 +3366,10 @@
// We split critical edges, so we change the CFG. However, we do update
// many analyses if they are around.
AU.addPreservedID(LoopSimplifyID);
- AU.addPreserved<LoopInfo>();
AU.addPreserved("domfrontier");
+ AU.addRequired<LoopInfo>();
+ AU.addPreserved<LoopInfo>();
AU.addRequiredID(LoopSimplifyID);
AU.addRequired<DominatorTree>();
AU.addPreserved<DominatorTree>();
Copied: llvm/branches/Apple/Morbo/test/CodeGen/X86/multiple-loop-post-inc.ll (from r100699, llvm/trunk/test/CodeGen/X86/multiple-loop-post-inc.ll)
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/test/CodeGen/X86/multiple-loop-post-inc.ll?p2=llvm/branches/Apple/Morbo/test/CodeGen/X86/multiple-loop-post-inc.ll&p1=llvm/trunk/test/CodeGen/X86/multiple-loop-post-inc.ll&r1=100699&r2=101103&rev=101103&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/multiple-loop-post-inc.ll (original)
+++ llvm/branches/Apple/Morbo/test/CodeGen/X86/multiple-loop-post-inc.ll Mon Apr 12 19:21:38 2010
@@ -275,3 +275,30 @@
return: ; preds = %entry
ret void
}
+
+; Codegen shouldn't crash on this testcase.
+
+define void @bar(i32 %a, i32 %b) nounwind {
+entry: ; preds = %bb1, %entry, %for.end204
+ br label %outer
+
+outer: ; preds = %bb1, %entry
+ %i6 = phi i32 [ %storemerge171, %bb1 ], [ %a, %entry ] ; <i32> [#uses=2]
+ %storemerge171 = add i32 %i6, 1 ; <i32> [#uses=1]
+ br label %inner
+
+inner: ; preds = %bb0, %if.end275
+ %i8 = phi i32 [ %a, %outer ], [ %indvar.next159, %bb0 ] ; <i32> [#uses=2]
+ %t338 = load i32* undef ; <i32> [#uses=1]
+ %t191 = mul i32 %i8, %t338 ; <i32> [#uses=1]
+ %t179 = add i32 %i6, %t191 ; <i32> [#uses=1]
+ br label %bb0
+
+bb0: ; preds = %for.body332
+ %indvar.next159 = add i32 %i8, 1 ; <i32> [#uses=1]
+ br i1 undef, label %bb1, label %inner
+
+bb1: ; preds = %bb0, %outer
+ %midx.4 = phi i32 [ %t179, %bb0 ] ; <i32> [#uses=0]
+ br label %outer
+}
Modified: llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/quadradic-exit-value.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/quadradic-exit-value.ll?rev=101103&r1=101102&r2=101103&view=diff
==============================================================================
--- llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/quadradic-exit-value.ll (original)
+++ llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/quadradic-exit-value.ll Mon Apr 12 19:21:38 2010
@@ -1,4 +1,4 @@
-; RUN: opt < %s -analyze -iv-users | grep {\{1,+,3,+,2\}<%loop> (post-inc)}
+; RUN: opt < %s -analyze -iv-users | grep {\{1,+,3,+,2\}<%loop> (post-inc with loop %loop)}
; The value of %r is dependent on a polynomial iteration expression.
Copied: llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/uglygep.ll (from r100884, llvm/trunk/test/Transforms/LoopStrengthReduce/uglygep.ll)
URL: http://llvm.org/viewvc/llvm-project/llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/uglygep.ll?p2=llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/uglygep.ll&p1=llvm/trunk/test/Transforms/LoopStrengthReduce/uglygep.ll&r1=100884&r2=101103&rev=101103&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/LoopStrengthReduce/uglygep.ll (original)
+++ llvm/branches/Apple/Morbo/test/Transforms/LoopStrengthReduce/uglygep.ll Mon Apr 12 19:21:38 2010
@@ -35,3 +35,33 @@
store i8 undef, i8* %t9
br label %bb14
}
+
+define fastcc void @TransformLine() nounwind {
+bb:
+ br label %loop0
+
+loop0: ; preds = %loop0, %bb
+ %i0 = phi i32 [ %i0.next, %loop0 ], [ 0, %bb ] ; <i32> [#uses=2]
+ %i0.next = add i32 %i0, 1 ; <i32> [#uses=1]
+ br i1 false, label %loop0, label %bb0
+
+bb0: ; preds = %loop0
+ br label %loop1
+
+loop1: ; preds = %bb5, %bb0
+ %i1 = phi i32 [ 0, %bb0 ], [ %i1.next, %bb5 ] ; <i32> [#uses=4]
+ %t0 = add i32 %i0, %i1 ; <i32> [#uses=1]
+ br i1 false, label %bb2, label %bb6
+
+bb2: ; preds = %loop1
+ br i1 true, label %bb6, label %bb5
+
+bb5: ; preds = %bb2
+ %i1.next = add i32 %i1, 1 ; <i32> [#uses=1]
+ br i1 true, label %bb6, label %loop1
+
+bb6: ; preds = %bb5, %bb2, %loop1
+ %p8 = phi i32 [ %t0, %bb5 ], [ undef, %loop1 ], [ undef, %bb2 ] ; <i32> [#uses=0]
+ %p9 = phi i32 [ undef, %bb5 ], [ %i1, %loop1 ], [ %i1, %bb2 ] ; <i32> [#uses=0]
+ unreachable
+}
More information about the llvm-branch-commits
mailing list