[llvm] [polly] [SCEV] Introduce SCEVUse, use it instead of const SCEV * (NFCI) (WIP). (PR #91961)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 13:48:36 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/91961
>From 216b44fd1b7822e48adca5e2591537a5a99d0d91 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 18 Feb 2026 12:36:13 +0000
Subject: [PATCH 1/3] [SCEV] Introduce SCEVUse wrapper type (NFC)
Add SCEVUse as a PointerIntPair wrapper around const SCEV * to prepare
for storing additional per-use information.
This commit contains the mechanical changes of adding an intial SCEVUse
wrapper and updating all relevant interfaces to take SCEVUse. Note that
currently the integer part is never set, and all SCEVUses are
considered canonical.
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 168 ++++++++++-
.../Analysis/ScalarEvolutionExpressions.h | 135 +++++----
.../Analysis/ScalarEvolutionPatternMatch.h | 16 +-
.../llvm/Transforms/Scalar/NaryReassociate.h | 15 +-
llvm/lib/Analysis/Delinearization.cpp | 6 +-
llvm/lib/Analysis/ScalarEvolution.cpp | 266 +++++++++++-------
llvm/lib/Analysis/ScalarEvolutionDivision.cpp | 4 +-
.../Analysis/ScalarEvolutionNormalization.cpp | 2 +-
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 2 +-
llvm/lib/Transforms/Scalar/IndVarSimplify.cpp | 5 +-
.../Scalar/InductiveRangeCheckElimination.cpp | 34 ++-
.../Transforms/Scalar/LoopDataPrefetch.cpp | 7 +-
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 9 +-
.../Transforms/Scalar/LoopIdiomRecognize.cpp | 8 +-
.../lib/Transforms/Scalar/LoopPredication.cpp | 9 +-
.../lib/Transforms/Scalar/NaryReassociate.cpp | 48 ++--
.../Scalar/StraightLineStrengthReduce.cpp | 11 +-
.../Vectorize/LoopVectorizationLegality.cpp | 3 +-
polly/include/polly/Support/ScopHelper.h | 3 +-
polly/lib/Support/ScopHelper.cpp | 24 +-
20 files changed, 500 insertions(+), 275 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 6aff53a1e7b70..b21ea762094ec 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -66,6 +66,115 @@ enum SCEVTypes : unsigned short;
LLVM_ABI extern bool VerifySCEV;
+class SCEV;
+
+struct SCEVUse : public PointerIntPair<const SCEV *, 3> {
+ SCEVUse() : PointerIntPair() { setFromOpaqueValue(nullptr); }
+ SCEVUse(const SCEV *S) : PointerIntPair() { setFromOpaqueValue((void *)S); }
+ SCEVUse(const SCEV *S, int Flags) : PointerIntPair(S, Flags) {}
+
+ operator const SCEV *() const { return getPointer(); }
+ const SCEV *operator->() const { return getPointer(); }
+ const SCEV *operator->() { return getPointer(); }
+
+ void *getRawPointer() const { return getOpaqueValue(); }
+
+ bool isCanonical() const;
+
+ const SCEV *getCanonical() const { return getPointer(); }
+
+ unsigned getFlags() const { return getInt(); }
+
+ bool operator==(const SCEVUse &RHS) const {
+ assert(isCanonical() && RHS.isCanonical());
+ return getRawPointer() == RHS.getRawPointer();
+ }
+
+ bool operator==(const SCEV *RHS) const { return getRawPointer() == RHS; }
+
+ /// Print out the internal representation of this scalar to the specified
+ /// stream. This should really only be used for debugging purposes.
+ void print(raw_ostream &OS) const;
+
+ /// This method is used for debugging.
+ void dump() const;
+
+ friend hash_code hash_value(const SCEVUse &U) {
+ return hash_value(U.getRawPointer());
+ }
+};
+
+/// Provide PointerLikeTypeTraits for SCEVUse, so it can be used with
+/// SmallPtrSet, among others.
+template <> struct PointerLikeTypeTraits<SCEVUse> {
+ static inline void *getAsVoidPointer(SCEVUse U) { return U.getOpaqueValue(); }
+ static inline SCEVUse getFromVoidPointer(void *P) {
+ SCEVUse U;
+ U.setFromOpaqueValue(P);
+ return U;
+ }
+
+ /// The Low bits are used by the PointerIntPair.
+ static constexpr int NumLowBitsAvailable = 0;
+};
+
+template <> struct DenseMapInfo<SCEVUse> {
+ static constexpr uintptr_t Log2MaxAlign = 12;
+
+ static inline SCEVUse getEmptyKey() {
+ uintptr_t Val = static_cast<uintptr_t>(-1);
+ Val <<= Log2MaxAlign;
+ return PointerLikeTypeTraits<SCEVUse>::getFromVoidPointer((void *)Val);
+ }
+
+ static inline SCEVUse getTombstoneKey() {
+ uintptr_t Val = static_cast<uintptr_t>(-2);
+ Val <<= Log2MaxAlign;
+ return PointerLikeTypeTraits<SCEVUse>::getFromVoidPointer((void *)Val);
+ }
+
+ static unsigned getHashValue(SCEVUse U) { return hash_value(U); }
+
+ static bool isEqual(const SCEVUse LHS, const SCEVUse RHS) {
+ return LHS.getRawPointer() == RHS.getRawPointer();
+ }
+};
+
+inline bool SCEVUse::isCanonical() const {
+ assert(getInt() == 0 && "Currently must always be canonical");
+ return getCanonical() == getPointer();
+}
+
+template <typename To> [[nodiscard]] inline decltype(auto) dyn_cast(SCEVUse U) {
+ assert(detail::isPresent(U.getPointer()) &&
+ "dyn_cast on a non-existent value");
+ return CastInfo<To, const SCEV *>::doCastIfPossible(U.getPointer());
+}
+
+template <typename To>
+[[nodiscard]] inline decltype(auto) dyn_cast_if_present(SCEVUse U) {
+ assert(detail::isPresent(U.getPointer()) &&
+ "dyn_cast on a non-existent value");
+ return CastInfo<To, const SCEV *>::doCastIfPossible(U.getPointer());
+}
+
+template <typename To> [[nodiscard]] inline decltype(auto) cast(SCEVUse U) {
+ assert(detail::isPresent(U.getPointer()) &&
+ "dyn_cast on a non-existent value");
+ return CastInfo<To, const SCEV *>::doCast(U.getPointer());
+}
+
+template <typename To> [[nodiscard]] inline bool isa(SCEVUse U) {
+ return CastInfo<To, const SCEV *>::isPossible(U.getPointer());
+}
+
+template <class X> auto dyn_cast_or_null(SCEVUse U) {
+ const SCEV *Val = U.getPointer();
+ if (!detail::isPresent(Val))
+ return CastInfo<X, const SCEV *>::castFailed();
+ return CastInfo<X, const SCEV *>::doCastIfPossible(detail::unwrapValue(Val));
+}
+
/// This class represents an analyzed expression in the program. These are
/// opaque objects that the client is not allowed to do much with directly.
///
@@ -144,7 +253,7 @@ class SCEV : public FoldingSetNode {
LLVM_ABI Type *getType() const;
/// Return operands of this SCEV expression.
- LLVM_ABI ArrayRef<const SCEV *> operands() const;
+ LLVM_ABI ArrayRef<SCEVUse> operands() const;
/// Return true if the expression is a constant zero.
LLVM_ABI bool isZero() const;
@@ -554,6 +663,7 @@ class ScalarEvolution {
/// Notify this ScalarEvolution that \p User directly uses SCEVs in \p Ops.
LLVM_ABI void registerUser(const SCEV *User, ArrayRef<const SCEV *> Ops);
+ LLVM_ABI void registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops);
/// Return true if the SCEV expression contains an undef value.
LLVM_ABI bool containsUndefs(const SCEV *S) const;
@@ -592,34 +702,65 @@ class ScalarEvolution {
unsigned Depth = 0);
LLVM_ABI const SCEV *getCastExpr(SCEVTypes Kind, const SCEV *Op, Type *Ty);
LLVM_ABI const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty);
- LLVM_ABI const SCEV *getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
+
+ LLVM_ABI const SCEV *getAddExpr(ArrayRef<const SCEV *> Ops,
+ SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
+ unsigned Depth = 0) {
+ SmallVector<SCEVUse> NewOps;
+ for (auto *Op : Ops)
+ NewOps.emplace_back(Op);
+ return getAddExpr(NewOps, Flags, Depth);
+ }
+
+ LLVM_ABI const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS,
+ SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
+ unsigned Depth = 0) {
+ return getAddExpr(SCEVUse(LHS), SCEVUse(RHS), Flags, Depth);
+ }
+
+ LLVM_ABI const SCEV *getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
unsigned Depth = 0);
- const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS,
+ const SCEV *getAddExpr(SCEVUse LHS, SCEVUse RHS,
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
unsigned Depth = 0) {
- SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
+ SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
return getAddExpr(Ops, Flags, Depth);
}
- const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2,
+ const SCEV *getAddExpr(SCEVUse Op0, SCEVUse Op1, SCEVUse Op2,
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
unsigned Depth = 0) {
- SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2};
+ SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
return getAddExpr(Ops, Flags, Depth);
}
- LLVM_ABI const SCEV *getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
+ LLVM_ABI const SCEV *getMulExpr(ArrayRef<const SCEV *> Ops,
+ SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
+ unsigned Depth = 0) {
+ SmallVector<SCEVUse> NewOps;
+ for (auto *Op : Ops)
+ NewOps.emplace_back(Op);
+ return getMulExpr(NewOps, Flags, Depth);
+ }
+
+ LLVM_ABI const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS,
+ SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
+ unsigned Depth = 0) {
+ return getMulExpr(SCEVUse(LHS), SCEVUse(RHS), Flags, Depth);
+ }
+
+ LLVM_ABI const SCEV *getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
unsigned Depth = 0);
- const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS,
+ const SCEV *getMulExpr(SCEVUse LHS, SCEVUse RHS,
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
unsigned Depth = 0) {
- SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
+ SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
return getMulExpr(Ops, Flags, Depth);
}
- const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2,
+ const SCEV *getMulExpr(SCEVUse Op0, SCEVUse Op1, SCEVUse Op2,
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
unsigned Depth = 0) {
- SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2};
+ SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
return getMulExpr(Ops, Flags, Depth);
}
LLVM_ABI const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS);
@@ -2315,11 +2456,11 @@ class ScalarEvolution {
bool canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride, bool IsSigned);
/// Get add expr already created or create a new one.
- const SCEV *getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
+ const SCEV *getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
SCEV::NoWrapFlags Flags);
/// Get mul expr already created or create a new one.
- const SCEV *getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
+ const SCEV *getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
SCEV::NoWrapFlags Flags);
// Get addrec expr already created or create a new one.
@@ -2337,6 +2478,7 @@ class ScalarEvolution {
/// Look for a SCEV expression with type `SCEVType` and operands `Ops` in
/// `UniqueSCEVs`. Return if found, else nullptr.
SCEV *findExistingSCEVInCache(SCEVTypes SCEVType, ArrayRef<const SCEV *> Ops);
+ SCEV *findExistingSCEVInCache(SCEVTypes SCEVType, ArrayRef<SCEVUse> Ops);
/// Get reachable blocks in this function, making limited use of SCEV
/// reasoning about conditions.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 2fd25dbb9062d..be81f2b456ccd 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -58,6 +58,13 @@ enum SCEVTypes : unsigned short {
scCouldNotCompute
};
+inline SmallVector<const SCEV *> toSCEV(ArrayRef<SCEVUse> Ops) {
+ SmallVector<const SCEV *> NewOps;
+ for (auto O : Ops)
+ NewOps.push_back(O);
+ return NewOps;
+}
+
/// This class represents a constant integer value.
class SCEVConstant : public SCEV {
friend class ScalarEvolution;
@@ -94,9 +101,9 @@ class SCEVVScale : public SCEV {
static bool classof(const SCEV *S) { return S->getSCEVType() == scVScale; }
};
-inline unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) {
+inline unsigned short computeExpressionSize(ArrayRef<SCEVUse> Args) {
APInt Size(16, 1);
- for (const auto *Arg : Args)
+ for (const SCEV *Arg : Args)
Size = Size.uadd_sat(APInt(16, Arg->getExpressionSize()));
return (unsigned short)Size.getZExtValue();
}
@@ -104,19 +111,19 @@ inline unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) {
/// This is the base class for unary cast operator classes.
class SCEVCastExpr : public SCEV {
protected:
- const SCEV *Op;
+ SCEVUse Op;
Type *Ty;
LLVM_ABI SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
- const SCEV *op, Type *ty);
+ SCEVUse op, Type *ty);
public:
- const SCEV *getOperand() const { return Op; }
- const SCEV *getOperand(unsigned i) const {
+ SCEVUse getOperand() const { return Op; }
+ SCEVUse getOperand(unsigned i) const {
assert(i == 0 && "Operand index out of range!");
return Op;
}
- ArrayRef<const SCEV *> operands() const { return Op; }
+ ArrayRef<SCEVUse> operands() const { return Op; }
size_t getNumOperands() const { return 1; }
Type *getType() const { return Ty; }
@@ -133,11 +140,12 @@ class SCEVCastExpr : public SCEV {
class SCEVPtrToIntExpr : public SCEVCastExpr {
friend class ScalarEvolution;
- SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op, Type *ITy);
+ SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, SCEVUse Op, Type *ITy);
public:
/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const SCEV *S) { return S->getSCEVType() == scPtrToInt; }
+ static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
};
/// This class represents a cast from a pointer to a pointer-sized integer
@@ -156,7 +164,7 @@ class SCEVPtrToAddrExpr : public SCEVCastExpr {
class SCEVIntegralCastExpr : public SCEVCastExpr {
protected:
LLVM_ABI SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
- const SCEV *op, Type *ty);
+ SCEVUse op, Type *ty);
public:
/// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -171,7 +179,7 @@ class SCEVIntegralCastExpr : public SCEVCastExpr {
class SCEVTruncateExpr : public SCEVIntegralCastExpr {
friend class ScalarEvolution;
- SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty);
+ SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op, Type *ty);
public:
/// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -183,7 +191,7 @@ class SCEVTruncateExpr : public SCEVIntegralCastExpr {
class SCEVZeroExtendExpr : public SCEVIntegralCastExpr {
friend class ScalarEvolution;
- SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty);
+ SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op, Type *ty);
public:
/// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -197,7 +205,7 @@ class SCEVZeroExtendExpr : public SCEVIntegralCastExpr {
class SCEVSignExtendExpr : public SCEVIntegralCastExpr {
friend class ScalarEvolution;
- SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty);
+ SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op, Type *ty);
public:
/// Methods for support type inquiry through isa, cast, and dyn_cast:
@@ -214,25 +222,23 @@ class SCEVNAryExpr : public SCEV {
// arrays with its SCEVAllocator, so this class just needs a simple
// pointer rather than a more elaborate vector-like data structure.
// This also avoids the need for a non-trivial destructor.
- const SCEV *const *Operands;
+ SCEVUse const *Operands;
size_t NumOperands;
- SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
- const SCEV *const *O, size_t N)
+ SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, SCEVUse const *O,
+ size_t N)
: SCEV(ID, T, computeExpressionSize(ArrayRef(O, N))), Operands(O),
NumOperands(N) {}
public:
size_t getNumOperands() const { return NumOperands; }
- const SCEV *getOperand(unsigned i) const {
+ SCEVUse getOperand(unsigned i) const {
assert(i < NumOperands && "Operand index out of range!");
return Operands[i];
}
- ArrayRef<const SCEV *> operands() const {
- return ArrayRef(Operands, NumOperands);
- }
+ ArrayRef<SCEVUse> operands() const { return ArrayRef(Operands, NumOperands); }
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask = NoWrapMask) const {
return (NoWrapFlags)(SubclassData & Mask);
@@ -256,13 +262,14 @@ class SCEVNAryExpr : public SCEV {
S->getSCEVType() == scSequentialUMinExpr ||
S->getSCEVType() == scAddRecExpr;
}
+ static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
};
/// This node is the base class for n'ary commutative operators.
class SCEVCommutativeExpr : public SCEVNAryExpr {
protected:
SCEVCommutativeExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
- const SCEV *const *O, size_t N)
+ SCEVUse const *O, size_t N)
: SCEVNAryExpr(ID, T, O, N) {}
public:
@@ -283,11 +290,10 @@ class SCEVAddExpr : public SCEVCommutativeExpr {
Type *Ty;
- SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+ SCEVAddExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N)
: SCEVCommutativeExpr(ID, scAddExpr, O, N) {
- auto *FirstPointerTypedOp = find_if(operands(), [](const SCEV *Op) {
- return Op->getType()->isPointerTy();
- });
+ auto *FirstPointerTypedOp = find_if(
+ operands(), [](SCEVUse Op) { return Op->getType()->isPointerTy(); });
if (FirstPointerTypedOp != operands().end())
Ty = (*FirstPointerTypedOp)->getType();
else
@@ -299,13 +305,14 @@ class SCEVAddExpr : public SCEVCommutativeExpr {
/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const SCEV *S) { return S->getSCEVType() == scAddExpr; }
+ static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
};
/// This node represents multiplication of some number of SCEVs.
class SCEVMulExpr : public SCEVCommutativeExpr {
friend class ScalarEvolution;
- SCEVMulExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+ SCEVMulExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N)
: SCEVCommutativeExpr(ID, scMulExpr, O, N) {}
public:
@@ -313,30 +320,31 @@ class SCEVMulExpr : public SCEVCommutativeExpr {
/// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const SCEV *S) { return S->getSCEVType() == scMulExpr; }
+ static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
};
/// This class represents a binary unsigned division operation.
class SCEVUDivExpr : public SCEV {
friend class ScalarEvolution;
- std::array<const SCEV *, 2> Operands;
+ std::array<SCEVUse, 2> Operands;
- SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs)
+ SCEVUDivExpr(const FoldingSetNodeIDRef ID, SCEVUse lhs, SCEVUse rhs)
: SCEV(ID, scUDivExpr, computeExpressionSize({lhs, rhs})) {
Operands[0] = lhs;
Operands[1] = rhs;
}
public:
- const SCEV *getLHS() const { return Operands[0]; }
- const SCEV *getRHS() const { return Operands[1]; }
+ SCEVUse getLHS() const { return Operands[0]; }
+ SCEVUse getRHS() const { return Operands[1]; }
size_t getNumOperands() const { return 2; }
- const SCEV *getOperand(unsigned i) const {
+ SCEVUse getOperand(unsigned i) const {
assert((i == 0 || i == 1) && "Operand index out of range!");
return i == 0 ? getLHS() : getRHS();
}
- ArrayRef<const SCEV *> operands() const { return Operands; }
+ ArrayRef<SCEVUse> operands() const { return Operands; }
Type *getType() const {
// In most cases the types of LHS and RHS will be the same, but in some
@@ -364,25 +372,24 @@ class SCEVAddRecExpr : public SCEVNAryExpr {
const Loop *L;
- SCEVAddRecExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N,
+ SCEVAddRecExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N,
const Loop *l)
: SCEVNAryExpr(ID, scAddRecExpr, O, N), L(l) {}
public:
Type *getType() const { return getStart()->getType(); }
- const SCEV *getStart() const { return Operands[0]; }
+ SCEVUse getStart() const { return Operands[0]; }
const Loop *getLoop() const { return L; }
/// Constructs and returns the recurrence indicating how much this
/// expression steps by. If this is a polynomial of degree N, it
/// returns a chrec of degree N-1. We cannot determine whether
/// the step recurrence has self-wraparound.
- const SCEV *getStepRecurrence(ScalarEvolution &SE) const {
+ SCEVUse getStepRecurrence(ScalarEvolution &SE) const {
if (isAffine())
return getOperand(1);
- return SE.getAddRecExpr(
- SmallVector<const SCEV *, 3>(operands().drop_front()), getLoop(),
- FlagAnyWrap);
+ return SE.getAddRecExpr(toSCEV(operands().drop_front()), getLoop(),
+ FlagAnyWrap);
}
/// Return true if this represents an expression A + B*x where A
@@ -414,9 +421,9 @@ class SCEVAddRecExpr : public SCEVNAryExpr {
/// Return the value of this chain of recurrences at the specified iteration
/// number. Takes an explicit list of operands to represent an AddRec.
- LLVM_ABI static const SCEV *
- evaluateAtIteration(ArrayRef<const SCEV *> Operands, const SCEV *It,
- ScalarEvolution &SE);
+ LLVM_ABI static const SCEV *evaluateAtIteration(ArrayRef<SCEVUse> Operands,
+ const SCEV *It,
+ ScalarEvolution &SE);
/// Return the number of iterations of this loop that produce
/// values in the specified constant range. Another way of
@@ -449,7 +456,7 @@ class SCEVMinMaxExpr : public SCEVCommutativeExpr {
protected:
/// Note: Constructing subclasses via this constructor is allowed
SCEVMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
- const SCEV *const *O, size_t N)
+ SCEVUse const *O, size_t N)
: SCEVCommutativeExpr(ID, T, O, N) {
assert(isMinMaxType(T));
// Min and max never overflow
@@ -481,7 +488,7 @@ class SCEVMinMaxExpr : public SCEVCommutativeExpr {
class SCEVSMaxExpr : public SCEVMinMaxExpr {
friend class ScalarEvolution;
- SCEVSMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+ SCEVSMaxExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N)
: SCEVMinMaxExpr(ID, scSMaxExpr, O, N) {}
public:
@@ -493,7 +500,7 @@ class SCEVSMaxExpr : public SCEVMinMaxExpr {
class SCEVUMaxExpr : public SCEVMinMaxExpr {
friend class ScalarEvolution;
- SCEVUMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+ SCEVUMaxExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N)
: SCEVMinMaxExpr(ID, scUMaxExpr, O, N) {}
public:
@@ -505,7 +512,7 @@ class SCEVUMaxExpr : public SCEVMinMaxExpr {
class SCEVSMinExpr : public SCEVMinMaxExpr {
friend class ScalarEvolution;
- SCEVSMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+ SCEVSMinExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N)
: SCEVMinMaxExpr(ID, scSMinExpr, O, N) {}
public:
@@ -517,7 +524,7 @@ class SCEVSMinExpr : public SCEVMinMaxExpr {
class SCEVUMinExpr : public SCEVMinMaxExpr {
friend class ScalarEvolution;
- SCEVUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
+ SCEVUMinExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O, size_t N)
: SCEVMinMaxExpr(ID, scUMinExpr, O, N) {}
public:
@@ -544,7 +551,7 @@ class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
protected:
/// Note: Constructing subclasses via this constructor is allowed
SCEVSequentialMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
- const SCEV *const *O, size_t N)
+ SCEVUse const *O, size_t N)
: SCEVNAryExpr(ID, T, O, N) {
assert(isSequentialMinMaxType(T));
// Min and max never overflow
@@ -571,13 +578,14 @@ class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
static bool classof(const SCEV *S) {
return isSequentialMinMaxType(S->getSCEVType());
}
+ static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
};
/// This class represents a sequential/in-order unsigned minimum selection.
class SCEVSequentialUMinExpr : public SCEVSequentialMinMaxExpr {
friend class ScalarEvolution;
- SCEVSequentialUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O,
+ SCEVSequentialUMinExpr(const FoldingSetNodeIDRef ID, SCEVUse const *O,
size_t N)
: SCEVSequentialMinMaxExpr(ID, scSequentialUMinExpr, O, N) {}
@@ -715,7 +723,7 @@ template <typename SV> class SCEVTraversal {
case scUMinExpr:
case scSequentialUMinExpr:
case scAddRecExpr:
- for (const auto *Op : S->operands()) {
+ for (const SCEV *Op : S->operands()) {
push(Op);
if (Visitor.isDone())
break;
@@ -827,7 +835,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -837,7 +845,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -854,7 +862,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -866,7 +874,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -876,7 +884,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -886,7 +894,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -896,7 +904,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -906,7 +914,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
const SCEV *visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (const SCEV *Op : Expr->operands()) {
Operands.push_back(((SC *)this)->visit(Op));
Changed |= Op != Operands.back();
}
@@ -964,15 +972,18 @@ class SCEVLoopAddRecRewriter
}
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
- SmallVector<const SCEV *, 2> Operands;
- for (const SCEV *Op : Expr->operands())
- Operands.push_back(visit(Op));
-
const Loop *L = Expr->getLoop();
auto It = Map.find(L);
- if (It == Map.end())
- return SE.getAddRecExpr(Operands, L, Expr->getNoWrapFlags());
+ if (It == Map.end()) {
+ SmallVector<const SCEV *, 2> Operands2;
+ for (const SCEV *Op : Expr->operands())
+ Operands2.push_back(visit(Op));
+ return SE.getAddRecExpr(Operands2, L, Expr->getNoWrapFlags());
+ }
+ SmallVector<SCEVUse, 2> Operands;
+ for (const SCEV *Op : Expr->operands())
+ Operands.push_back(visit(Op));
return SCEVAddRecExpr::evaluateAtIteration(Operands, It->second, SE);
}
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
index 7b00d0109a68c..0b47357434782 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
@@ -13,6 +13,7 @@
#ifndef LLVM_ANALYSIS_SCALAREVOLUTIONPATTERNMATCH_H
#define LLVM_ANALYSIS_SCALAREVOLUTIONPATTERNMATCH_H
+#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
namespace llvm {
@@ -22,6 +23,11 @@ template <typename Pattern> bool match(const SCEV *S, const Pattern &P) {
return P.match(S);
}
+template <typename Pattern> bool match(const SCEVUse U, const Pattern &P) {
+ const SCEV *S = U.getPointer();
+ return const_cast<Pattern &>(P).match(S);
+}
+
template <typename Predicate> struct cst_pred_ty : public Predicate {
cst_pred_ty() = default;
cst_pred_ty(uint64_t V) : Predicate(V) {}
@@ -159,7 +165,8 @@ template <typename SCEVTy, typename Op0_t> struct SCEVUnaryExpr_match {
bool match(const SCEV *S) const {
auto *E = dyn_cast<SCEVTy>(S);
- return E && E->getNumOperands() == 1 && Op0.match(E->getOperand(0));
+ return E && E->getNumOperands() == 1 &&
+ Op0.match(E->getOperand(0).getPointer());
}
};
@@ -215,9 +222,10 @@ struct SCEVBinaryExpr_match {
auto *E = dyn_cast<SCEVTy>(S);
return E && E->getNumOperands() == 2 &&
- ((Op0.match(E->getOperand(0)) && Op1.match(E->getOperand(1))) ||
- (Commutable && Op0.match(E->getOperand(1)) &&
- Op1.match(E->getOperand(0))));
+ ((Op0.match(E->getOperand(0).getPointer()) &&
+ Op1.match(E->getOperand(1).getPointer())) ||
+ (Commutable && Op0.match(E->getOperand(1).getPointer()) &&
+ Op1.match(E->getOperand(0).getPointer())));
}
};
diff --git a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
index f0474bc4352e3..417801d470800 100644
--- a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
+++ b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
@@ -80,6 +80,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ValueHandle.h"
@@ -92,8 +93,6 @@ class DominatorTree;
class Function;
class GetElementPtrInst;
class Instruction;
-class ScalarEvolution;
-class SCEV;
class TargetLibraryInfo;
class TargetTransformInfo;
class Type;
@@ -114,7 +113,7 @@ class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> {
bool doOneIteration(Function &F);
// Reassociates I for better CSE.
- Instruction *tryReassociate(Instruction *I, const SCEV *&OrigSCEV);
+ Instruction *tryReassociate(Instruction *I, SCEVUse &OrigSCEV);
// Reassociate GEP for better CSE.
Instruction *tryReassociateGEP(GetElementPtrInst *GEP);
@@ -143,19 +142,18 @@ class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> {
Instruction *tryReassociateBinaryOp(Value *LHS, Value *RHS,
BinaryOperator *I);
// Rewrites I to (LHS op RHS) if LHS is computed already.
- Instruction *tryReassociatedBinaryOp(const SCEV *LHS, Value *RHS,
+ Instruction *tryReassociatedBinaryOp(SCEVUse LHS, Value *RHS,
BinaryOperator *I);
// Tries to match Op1 and Op2 by using V.
bool matchTernaryOp(BinaryOperator *I, Value *V, Value *&Op1, Value *&Op2);
// Gets SCEV for (LHS op RHS).
- const SCEV *getBinarySCEV(BinaryOperator *I, const SCEV *LHS,
- const SCEV *RHS);
+ SCEVUse getBinarySCEV(BinaryOperator *I, SCEVUse LHS, SCEVUse RHS);
// Returns the closest dominator of \c Dominatee that computes
// \c CandidateExpr. Returns null if not found.
- Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr,
+ Instruction *findClosestMatchingDominator(SCEVUse CandidateExpr,
Instruction *Dominatee);
// Try to match \p I as signed/unsigned Min/Max and reassociate it. \p
@@ -163,8 +161,7 @@ class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> {
// done or not. If reassociation was successful newly generated instruction is
// returned, otherwise nullptr.
template <typename PredT>
- Instruction *matchAndReassociateMinOrMax(Instruction *I,
- const SCEV *&OrigSCEV);
+ Instruction *matchAndReassociateMinOrMax(Instruction *I, SCEVUse &OrigSCEV);
// Reassociate Min/Max.
template <typename MaxMinT>
diff --git a/llvm/lib/Analysis/Delinearization.cpp b/llvm/lib/Analysis/Delinearization.cpp
index b8582b09025a6..15876ee720a38 100644
--- a/llvm/lib/Analysis/Delinearization.cpp
+++ b/llvm/lib/Analysis/Delinearization.cpp
@@ -135,7 +135,7 @@ struct SCEVCollectAddRecMultiplies {
bool follow(const SCEV *S) {
if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) {
bool HasAddRec = false;
- SmallVector<const SCEV *, 0> Operands;
+ SmallVector<SCEVUse, 0> Operands;
for (const SCEV *Op : Mul->operands()) {
const SCEVUnknown *Unknown = dyn_cast<SCEVUnknown>(Op);
if (Unknown && !isa<CallInst>(Unknown->getValue())) {
@@ -209,7 +209,7 @@ static bool findArrayDimensionsRec(ScalarEvolution &SE,
// End of recursion.
if (Last == 0) {
if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) {
- SmallVector<const SCEV *, 2> Qs;
+ SmallVector<SCEVUse, 2> Qs;
for (const SCEV *Op : M->operands())
if (!isa<SCEVConstant>(Op))
Qs.push_back(Op);
@@ -268,7 +268,7 @@ static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) {
return T;
if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) {
- SmallVector<const SCEV *, 2> Factors;
+ SmallVector<SCEVUse, 2> Factors;
for (const SCEV *Op : M->operands())
if (!isa<SCEVConstant>(Op))
Factors.push_back(Op);
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 937b5aa77d467..8cc118ed2e784 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -258,6 +258,22 @@ static cl::opt<bool> UseContextForNoWrapFlagInference(
// SCEV class definitions
//===----------------------------------------------------------------------===//
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void SCEVUse::dump() const {
+ print(dbgs());
+ dbgs() << '\n';
+}
+#endif
+
+void SCEVUse::print(raw_ostream &OS) const {
+ getPointer()->print(OS);
+ SCEV::NoWrapFlags Flags = static_cast<SCEV::NoWrapFlags>(getInt());
+ if (Flags & SCEV::FlagNUW)
+ OS << "(u nuw)";
+ if (Flags & SCEV::FlagNSW)
+ OS << "(u nsw)";
+}
+
//===----------------------------------------------------------------------===//
// Implementation of the SCEV class.
//
@@ -417,7 +433,7 @@ Type *SCEV::getType() const {
llvm_unreachable("Unknown SCEV kind!");
}
-ArrayRef<const SCEV *> SCEV::operands() const {
+ArrayRef<SCEVUse> SCEV::operands() const {
switch (getSCEVType()) {
case scConstant:
case scVScale:
@@ -516,7 +532,7 @@ const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
}
SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
- const SCEV *op, Type *ty)
+ SCEVUse op, Type *ty)
: SCEV(ID, SCEVTy, computeExpressionSize(op)), Op(op), Ty(ty) {}
SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
@@ -526,7 +542,7 @@ SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
"Must be a non-bit-width-changing pointer-to-integer cast!");
}
-SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op,
+SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, SCEVUse Op,
Type *ITy)
: SCEVCastExpr(ID, scPtrToInt, Op, ITy) {
assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
@@ -534,26 +550,26 @@ SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op,
}
SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
- SCEVTypes SCEVTy, const SCEV *op,
+ SCEVTypes SCEVTy, SCEVUse op,
Type *ty)
: SCEVCastExpr(ID, SCEVTy, op, ty) {}
-SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op,
+SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
Type *ty)
: SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
"Cannot truncate non-integer value!");
}
-SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
- const SCEV *op, Type *ty)
+SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
+ Type *ty)
: SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
"Cannot zero extend non-integer value!");
}
-SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
- const SCEV *op, Type *ty)
+SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
+ Type *ty)
: SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
"Cannot sign extend non-integer value!");
@@ -748,8 +764,8 @@ CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
case scSMinExpr:
case scUMinExpr:
case scSequentialUMinExpr: {
- ArrayRef<const SCEV *> LOps = LHS->operands();
- ArrayRef<const SCEV *> ROps = RHS->operands();
+ ArrayRef<SCEVUse> LOps = LHS->operands();
+ ArrayRef<SCEVUse> ROps = RHS->operands();
// Lexicographically compare n-ary-like expressions.
unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
@@ -757,7 +773,8 @@ CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
return (int)LNumOps - (int)RNumOps;
for (unsigned i = 0; i != LNumOps; ++i) {
- auto X = CompareSCEVComplexity(LI, LOps[i], ROps[i], DT, Depth + 1);
+ auto X = CompareSCEVComplexity(LI, LOps[i].getPointer(),
+ ROps[i].getPointer(), DT, Depth + 1);
if (X != 0)
return X;
}
@@ -779,28 +796,27 @@ CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
/// results from this routine. In other words, we don't want the results of
/// this to depend on where the addresses of various SCEV objects happened to
/// land in memory.
-static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
- LoopInfo *LI, DominatorTree &DT) {
+static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
+ DominatorTree &DT) {
if (Ops.size() < 2) return; // Noop
// Whether LHS has provably less complexity than RHS.
- auto IsLessComplex = [&](const SCEV *LHS, const SCEV *RHS) {
+ auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
return Complexity && *Complexity < 0;
};
if (Ops.size() == 2) {
// This is the common case, which also happens to be trivially simple.
// Special case it.
- const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
+ SCEVUse &LHS = Ops[0], &RHS = Ops[1];
if (IsLessComplex(RHS, LHS))
std::swap(LHS, RHS);
return;
}
// Do the rough sort by complexity.
- llvm::stable_sort(Ops, [&](const SCEV *LHS, const SCEV *RHS) {
- return IsLessComplex(LHS, RHS);
- });
+ llvm::stable_sort(
+ Ops, [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
// Now that we are sorted by complexity, group elements of the same
// complexity. Note that this is, at worst, N^2, but the vector is likely to
@@ -825,7 +841,7 @@ static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
/// least HugeExprThreshold nodes).
-static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
+static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
return any_of(Ops, [](const SCEV *S) {
return S->getExpressionSize() >= HugeExprThreshold;
});
@@ -842,7 +858,7 @@ static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
static const SCEV *
constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
- SmallVectorImpl<const SCEV *> &Ops, FoldT Fold,
+ SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
const SCEVConstant *Folded = nullptr;
for (unsigned Idx = 0; Idx < Ops.size();) {
@@ -997,11 +1013,11 @@ const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
return evaluateAtIteration(operands(), It, SE);
}
-const SCEV *
-SCEVAddRecExpr::evaluateAtIteration(ArrayRef<const SCEV *> Operands,
- const SCEV *It, ScalarEvolution &SE) {
+const SCEV *SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
+ const SCEV *It,
+ ScalarEvolution &SE) {
assert(Operands.size() > 0);
- const SCEV *Result = Operands[0];
+ const SCEV *Result = Operands[0].getPointer();
for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
// The computation is correct in the face of overflow provided that the
// multiplication is performed _after_ the evaluation of the binomial
@@ -1010,7 +1026,8 @@ SCEVAddRecExpr::evaluateAtIteration(ArrayRef<const SCEV *> Operands,
if (isa<SCEVCouldNotCompute>(Coeff))
return Coeff;
- Result = SE.getAddExpr(Result, SE.getMulExpr(Operands[i], Coeff));
+ Result =
+ SE.getAddExpr(Result, SE.getMulExpr(Operands[i].getPointer(), Coeff));
}
return Result;
}
@@ -1057,9 +1074,9 @@ class SCEVCastSinkingRewriter
// implementation drops.
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
- Operands.push_back(visit(Op));
- Changed |= Op != Operands.back();
+ for (SCEVUse Op : Expr->operands()) {
+ Operands.push_back(visit(Op.getPointer()));
+ Changed |= Op.getPointer() != Operands.back();
}
return !Changed ? Expr : SE.getAddExpr(Operands, Expr->getNoWrapFlags());
}
@@ -1067,9 +1084,9 @@ class SCEVCastSinkingRewriter
const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
- Operands.push_back(visit(Op));
- Changed |= Op != Operands.back();
+ for (SCEVUse Op : Expr->operands()) {
+ Operands.push_back(visit(Op.getPointer()));
+ Changed |= Op.getPointer() != Operands.back();
}
return !Changed ? Expr : SE.getMulExpr(Operands, Expr->getNoWrapFlags());
}
@@ -1820,7 +1837,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
// If the addition does not unsign overflow then we can, by definition,
// commute the zero extension with the addition operation.
SmallVector<const SCEV *, 4> Ops;
- for (const auto *Op : SA->operands())
+ for (SCEVUse Op : SA->operands())
Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
return getAddExpr(Ops, SCEV::FlagNUW, Depth + 1);
}
@@ -1853,7 +1870,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
// If the multiply does not unsign overflow then we can, by definition,
// commute the zero extension with the multiply operation.
SmallVector<const SCEV *, 4> Ops;
- for (const auto *Op : SM->operands())
+ for (SCEVUse Op : SM->operands())
Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
return getMulExpr(Ops, SCEV::FlagNUW, Depth + 1);
}
@@ -1890,7 +1907,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
if (isa<SCEVUMinExpr>(Op) || isa<SCEVUMaxExpr>(Op)) {
auto *MinMax = cast<SCEVMinMaxExpr>(Op);
SmallVector<const SCEV *, 4> Operands;
- for (auto *Operand : MinMax->operands())
+ for (SCEVUse Operand : MinMax->operands())
Operands.push_back(getZeroExtendExpr(Operand, Ty));
if (isa<SCEVUMinExpr>(MinMax))
return getUMinExpr(Operands);
@@ -1901,7 +1918,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Op)) {
assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
SmallVector<const SCEV *, 4> Operands;
- for (auto *Operand : MinMax->operands())
+ for (SCEVUse Operand : MinMax->operands())
Operands.push_back(getZeroExtendExpr(Operand, Ty));
return getUMinExpr(Operands, /*Sequential*/ true);
}
@@ -1991,7 +2008,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
// If the addition does not sign overflow then we can, by definition,
// commute the sign extension with the addition operation.
SmallVector<const SCEV *, 4> Ops;
- for (const auto *Op : SA->operands())
+ for (SCEVUse Op : SA->operands())
Ops.push_back(getSignExtendExpr(Op, Ty, Depth + 1));
return getAddExpr(Ops, SCEV::FlagNSW, Depth + 1);
}
@@ -2161,7 +2178,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
if (isa<SCEVSMinExpr>(Op) || isa<SCEVSMaxExpr>(Op)) {
auto *MinMax = cast<SCEVMinMaxExpr>(Op);
SmallVector<const SCEV *, 4> Operands;
- for (auto *Operand : MinMax->operands())
+ for (SCEVUse Operand : MinMax->operands())
Operands.push_back(getSignExtendExpr(Operand, Ty));
if (isa<SCEVSMinExpr>(MinMax))
return getSMinExpr(Operands);
@@ -2174,7 +2191,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
- registerUser(S, { Op });
+ registerUser(S, ArrayRef<SCEVUse>(SCEVUse(Op)));
return S;
}
@@ -2266,12 +2283,12 @@ const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
/// may be exposed. This helps getAddRecExpr short-circuit extra work in
/// the common case where no interesting opportunities are present, and
/// is also used as a check to avoid infinite recursion.
-static bool
-CollectAddOperandsWithScales(SmallDenseMap<const SCEV *, APInt, 16> &M,
- SmallVectorImpl<const SCEV *> &NewOps,
- APInt &AccumulatedConstant,
- ArrayRef<const SCEV *> Ops, const APInt &Scale,
- ScalarEvolution &SE) {
+static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
+ SmallVectorImpl<SCEVUse> &NewOps,
+ APInt &AccumulatedConstant,
+ ArrayRef<SCEVUse> Ops,
+ const APInt &Scale,
+ ScalarEvolution &SE) {
bool Interesting = false;
// Iterate over the add operands. They are sorted, with constants first.
@@ -2294,9 +2311,11 @@ CollectAddOperandsWithScales(SmallDenseMap<const SCEV *, APInt, 16> &M,
if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
// A multiplication of a constant with another add; recurse.
const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
- Interesting |=
- CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
- Add->operands(), NewScale, SE);
+ SmallVector<SCEVUse> Ops2;
+ for (const SCEV *S : Add->operands())
+ Ops2.emplace_back(S);
+ Interesting |= CollectAddOperandsWithScales(
+ M, NewOps, AccumulatedConstant, Ops2, NewScale, SE);
} else {
// A multiplication of a constant with some other value. Update
// the map.
@@ -2314,8 +2333,7 @@ CollectAddOperandsWithScales(SmallDenseMap<const SCEV *, APInt, 16> &M,
}
} else {
// An ordinary operand. Update the map.
- std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
- M.insert({Ops[i], Scale});
+ auto Pair = M.insert({Ops[i], Scale});
if (Pair.second) {
NewOps.push_back(Pair.first->first);
} else {
@@ -2456,7 +2474,7 @@ ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
// can't-overflow flags for the operation if possible.
static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
SCEVTypes Type,
- ArrayRef<const SCEV *> Ops,
+ const ArrayRef<const SCEV *> Ops,
SCEV::NoWrapFlags Flags) {
using namespace std::placeholders;
@@ -2542,7 +2560,7 @@ bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
}
/// Get a canonical add expression, or something simpler if possible.
-const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
+const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
SCEV::NoWrapFlags OrigFlags,
unsigned Depth) {
assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
@@ -2570,8 +2588,11 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
unsigned Idx = isa<SCEVConstant>(Ops[0]) ? 1 : 0;
// Delay expensive flag strengthening until necessary.
- auto ComputeFlags = [this, OrigFlags](ArrayRef<const SCEV *> Ops) {
- return StrengthenNoWrapFlags(this, scAddExpr, Ops, OrigFlags);
+ auto ComputeFlags = [this, OrigFlags](ArrayRef<SCEVUse> Ops) {
+ SmallVector<const SCEV *> Ops_;
+ for (auto U : Ops)
+ Ops_.push_back(U);
+ return StrengthenNoWrapFlags(this, scAddExpr, Ops_, OrigFlags);
};
// Limit recursion calls depth.
@@ -2598,7 +2619,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
while (i+Count != e && Ops[i+Count] == Ops[i])
++Count;
// Merge the values into a multiply.
- const SCEV *Scale = getConstant(Ty, Count);
+ SCEVUse Scale = getConstant(Ty, Count);
const SCEV *Mul = getMulExpr(Scale, Ops[i], SCEV::FlagAnyWrap, Depth + 1);
if (Ops.size() == Count)
return Mul;
@@ -2622,7 +2643,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
if (auto *T = dyn_cast<SCEVTruncateExpr>(Ops[Idx]))
return T->getOperand()->getType();
if (const auto *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
- const auto *LastOp = Mul->getOperand(Mul->getNumOperands() - 1);
+ SCEVUse LastOp = Mul->getOperand(Mul->getNumOperands() - 1);
if (const auto *T = dyn_cast<SCEVTruncateExpr>(LastOp))
return T->getOperand()->getType();
}
@@ -2774,8 +2795,8 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
// operands multiplied by constant values.
if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
uint64_t BitWidth = getTypeSizeInBits(Ty);
- SmallDenseMap<const SCEV *, APInt, 16> M;
- SmallVector<const SCEV *, 8> NewOps;
+ SmallDenseMap<SCEVUse, APInt, 16> M;
+ SmallVector<SCEVUse, 8> NewOps;
APInt AccumulatedConstant(BitWidth, 0);
if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
Ops, APInt(BitWidth, 1), *this)) {
@@ -2900,7 +2921,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
// Scan all of the other operands to this add and add them to the vector if
// they are loop invariant w.r.t. the recurrence.
- SmallVector<const SCEV *, 8> LIOps;
+ SmallVector<SCEVUse, 8> LIOps;
const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
const Loop *AddRecLoop = AddRec->getLoop();
for (unsigned i = 0, e = Ops.size(); i != e; ++i)
@@ -2936,7 +2957,10 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
// in the outer scope.
SCEV::NoWrapFlags AddFlags = Flags;
if (AddFlags != SCEV::FlagAnyWrap) {
- auto *DefI = getDefiningScopeBound(LIOps);
+ SmallVector<const SCEV *> LIOps_;
+ for (auto U : LIOps)
+ LIOps_.push_back(U);
+ auto *DefI = getDefiningScopeBound(LIOps_);
auto *ReachI = &*AddRecLoop->getHeader()->begin();
if (!isGuaranteedToTransferExecutionTo(DefI, ReachI))
AddFlags = SCEV::FlagAnyWrap;
@@ -3008,9 +3032,8 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
}
-const SCEV *
-ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
- SCEV::NoWrapFlags Flags) {
+const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
+ SCEV::NoWrapFlags Flags) {
FoldingSetNodeID ID;
ID.AddInteger(scAddExpr);
for (const SCEV *Op : Ops)
@@ -3019,7 +3042,7 @@ ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
SCEVAddExpr *S =
static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
if (!S) {
- const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
+ SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
S = new (SCEVAllocator)
SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size());
@@ -3042,7 +3065,7 @@ ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
SCEVAddRecExpr *S =
static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
if (!S) {
- const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
+ SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
S = new (SCEVAllocator)
SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L);
@@ -3054,9 +3077,8 @@ ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
return S;
}
-const SCEV *
-ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
- SCEV::NoWrapFlags Flags) {
+const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
+ SCEV::NoWrapFlags Flags) {
FoldingSetNodeID ID;
ID.AddInteger(scMulExpr);
for (const SCEV *Op : Ops)
@@ -3065,7 +3087,7 @@ ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
SCEVMulExpr *S =
static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
if (!S) {
- const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
+ SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
O, Ops.size());
@@ -3131,7 +3153,7 @@ static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
}
/// Get a canonical multiply expression, or something simpler if possible.
-const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
+const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
SCEV::NoWrapFlags OrigFlags,
unsigned Depth) {
assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
@@ -3155,8 +3177,11 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
return Folded;
// Delay expensive flag strengthening until necessary.
- auto ComputeFlags = [this, OrigFlags](ArrayRef<const SCEV *> Ops) {
- return StrengthenNoWrapFlags(this, scMulExpr, Ops, OrigFlags);
+ auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
+ SmallVector<const SCEV *> Ops_;
+ for (auto U : Ops)
+ Ops_.push_back(U);
+ return StrengthenNoWrapFlags(this, scMulExpr, Ops_, OrigFlags);
};
// Limit recursion calls depth.
@@ -3195,8 +3220,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
SmallVector<const SCEV *, 4> NewOps;
bool AnyFolded = false;
for (const SCEV *AddOp : Add->operands()) {
- const SCEV *Mul = getMulExpr(Ops[0], AddOp, SCEV::FlagAnyWrap,
- Depth + 1);
+ const SCEV *Mul = getMulExpr(Ops[0], SCEVUse(AddOp),
+ SCEV::FlagAnyWrap, Depth + 1);
if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
NewOps.push_back(Mul);
}
@@ -3206,8 +3231,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
// Negation preserves a recurrence's no self-wrap property.
SmallVector<const SCEV *, 4> Operands;
for (const SCEV *AddRecOp : AddRec->operands())
- Operands.push_back(getMulExpr(Ops[0], AddRecOp, SCEV::FlagAnyWrap,
- Depth + 1));
+ Operands.push_back(getMulExpr(Ops[0], SCEVUse(AddRecOp),
+ SCEV::FlagAnyWrap, Depth + 1));
// Let M be the minimum representable signed value. AddRec with nsw
// multiplied by -1 can have signed overflow if and only if it takes a
// value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
@@ -3326,7 +3351,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
AddRec->getNoWrapFlags(ComputeFlags({Scale, AddRec}));
for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
- NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i),
+ NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i).getPointer(),
SCEV::FlagAnyWrap, Depth + 1));
if (hasFlags(Flags, SCEV::FlagNSW) && !hasFlags(Flags, SCEV::FlagNUW)) {
@@ -3639,7 +3664,7 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
LHS, RHS);
UniqueSCEVs.InsertNode(S, IP);
- registerUser(S, {LHS, RHS});
+ registerUser(S, ArrayRef<SCEVUse>({SCEVUse(LHS), SCEVUse(RHS)}));
return S;
}
@@ -3902,13 +3927,28 @@ SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
return UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
}
+SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
+ ArrayRef<SCEVUse> Ops) {
+ FoldingSetNodeID ID;
+ ID.AddInteger(SCEVType);
+ for (const SCEV *Op : Ops)
+ ID.AddPointer(Op);
+ void *IP = nullptr;
+ return UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
+}
+
const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
return getSMaxExpr(Op, getNegativeSCEV(Op, Flags));
}
-const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
- SmallVectorImpl<const SCEV *> &Ops) {
+const SCEV *
+ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
+ SmallVectorImpl<const SCEV *> &Ops_) {
+ SmallVector<SCEVUse> Ops;
+ for (const SCEV *U : Ops_)
+ Ops.push_back(U);
+
assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
if (Ops.size() == 1) return Ops[0];
@@ -3980,8 +4020,12 @@ const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
DeletedAny = true;
}
- if (DeletedAny)
- return getMinMaxExpr(Kind, Ops);
+ if (DeletedAny) {
+ SmallVector<const SCEV *> SU;
+ for (const SCEV *O : Ops)
+ SU.push_back(O);
+ return getMinMaxExpr(Kind, SU);
+ }
}
// Okay, check to see if the same value occurs in the operand list twice. If
@@ -4024,7 +4068,7 @@ const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
if (ExistingSCEV)
return ExistingSCEV;
- const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
+ SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
SCEV *S = new (SCEVAllocator)
SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
@@ -4062,8 +4106,11 @@ class SCEVSequentialMinMaxDeduplicatingVisitor final
return S;
auto *NAry = cast<SCEVNAryExpr>(S);
+ SmallVector<const SCEV *, 8> Operands;
+ for (SCEVUse U : NAry->operands())
+ Operands.push_back(U.getPointer());
SmallVector<const SCEV *> NewOps;
- bool Changed = visit(Kind, NAry->operands(), NewOps);
+ bool Changed = visit(Kind, Operands, NewOps);
if (!Changed)
return S;
@@ -4414,7 +4461,7 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
if (ExistingSCEV)
return ExistingSCEV;
- const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
+ SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
llvm::uninitialized_copy(Ops, O);
SCEV *S = new (SCEVAllocator)
SCEVSequentialMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
@@ -7408,7 +7455,7 @@ ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops,
if (!Bound || DT.dominates(Bound, DefI))
Bound = DefI;
} else {
- for (const auto *Op : S->operands())
+ for (SCEVUse Op : S->operands())
pushOp(Op);
}
}
@@ -10155,21 +10202,22 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
case scUMinExpr:
case scSMinExpr:
case scSequentialUMinExpr: {
- ArrayRef<const SCEV *> Ops = V->operands();
+ ArrayRef<SCEVUse> Ops = V->operands();
// Avoid performing the look-up in the common case where the specified
// expression has no loop-variant portions.
for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
- const SCEV *OpAtScope = getSCEVAtScope(Ops[i], L);
- if (OpAtScope != Ops[i]) {
+ const SCEV *OpAtScope = getSCEVAtScope(Ops[i].getPointer(), L);
+ if (OpAtScope != Ops[i].getPointer()) {
// Okay, at least one of these operands is loop variant but might be
// foldable. Build a new instance of the folded commutative expression.
SmallVector<const SCEV *, 8> NewOps;
NewOps.reserve(Ops.size());
- append_range(NewOps, Ops.take_front(i));
+ for (unsigned j = 0; j < i; ++j)
+ NewOps.push_back(Ops[j].getPointer());
NewOps.push_back(OpAtScope);
for (++i; i != e; ++i) {
- OpAtScope = getSCEVAtScope(Ops[i], L);
+ OpAtScope = getSCEVAtScope(Ops[i].getPointer(), L);
NewOps.push_back(OpAtScope);
}
@@ -11463,7 +11511,7 @@ ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
// work, try the following trick: if the a predicate is invariant for X, it
// is also invariant for umin(X, ...). So try to find something that works
// among subexpressions of MaxIter expressed as umin.
- for (auto *Op : UMin->operands())
+ for (SCEVUse Op : UMin->operands())
if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
Pred, LHS, RHS, L, CtxI, Op))
return LIP;
@@ -12779,9 +12827,9 @@ bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
if (P != ICmpInst::ICMP_SGT)
return false;
- auto GetOpFromSExt = [&](const SCEV *S) {
+ auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(S))
- return Ext->getOperand();
+ return Ext->getOperand().getPointer();
// TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
// the constant in some cases.
return S;
@@ -12813,8 +12861,8 @@ bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
if (!LHSAddExpr->hasNoSignedWrap())
return false;
- auto *LL = LHSAddExpr->getOperand(0);
- auto *LR = LHSAddExpr->getOperand(1);
+ SCEVUse LL = LHSAddExpr->getOperand(0);
+ SCEVUse LR = LHSAddExpr->getOperand(1);
auto *MinusOne = getMinusOne(RHS->getType());
// Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
@@ -14234,7 +14282,7 @@ ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
// This recurrence is variant w.r.t. L if any of its operands
// are variant.
- for (const auto *Op : AR->operands())
+ for (SCEVUse Op : AR->operands())
if (!isLoopInvariant(Op, L))
return LoopVariant;
@@ -14255,7 +14303,7 @@ ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
case scSMinExpr:
case scSequentialUMinExpr: {
bool HasVarying = false;
- for (const auto *Op : S->operands()) {
+ for (SCEVUse Op : S->operands()) {
LoopDisposition D = getLoopDisposition(Op, L);
if (D == LoopVariant)
return LoopVariant;
@@ -14681,7 +14729,7 @@ void ScalarEvolution::verify() const {
// Verify integrity of SCEV users.
for (const auto &S : UniqueSCEVs) {
- for (const auto *Op : S.operands()) {
+ for (SCEVUse Op : S.operands()) {
// We do not store dependencies of constants.
if (isa<SCEVConstant>(Op))
continue;
@@ -15107,7 +15155,8 @@ const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
continue;
ExitCount = getTruncateOrSignExtend(ExitCount, Step->getType());
- const SCEV *Add = getAddExpr(AddRecToCheck->getStart(), ExitCount);
+ const SCEV *Add =
+ getAddExpr(AddRecToCheck->getStart().getPointer(), ExitCount);
if (isKnownPredicate(CmpInst::ICMP_SLT, Add, AddRecToCheck->getStart()))
return nullptr;
}
@@ -15320,6 +15369,15 @@ void ScalarEvolution::registerUser(const SCEV *User,
SCEVUsers[Op].insert(User);
}
+void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
+ for (const SCEV *Op : Ops)
+ // We do not expect that forgetting cached data for SCEVConstants will ever
+ // open any prospects for sharpening or introduce any correctness issues,
+ // so we don't bother storing their dependencies.
+ if (!isa<SCEVConstant>(Op))
+ SCEVUsers[Op].insert(User);
+}
+
const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
const SCEV *Expr = SE.getSCEV(V);
return getPredicatedSCEV(Expr);
@@ -16117,14 +16175,14 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
SE.getAddExpr(Expr->getOperand(1), Expr->getOperand(2));
if (const SCEV *Rewritten = RewriteSubtraction(Add))
return SE.getAddExpr(
- Expr->getOperand(0), Rewritten,
+ Expr->getOperand(0).getPointer(), Rewritten,
ScalarEvolution::maskFlags(Expr->getNoWrapFlags(), FlagMask));
if (const SCEV *S = Map.lookup(Add))
- return SE.getAddExpr(Expr->getOperand(0), S);
+ return SE.getAddExpr(Expr->getOperand(0).getPointer(), S);
}
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (SCEVUse Op : Expr->operands()) {
Operands.push_back(
SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(Op));
Changed |= Op != Operands.back();
@@ -16140,7 +16198,7 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
SmallVector<const SCEV *, 2> Operands;
bool Changed = false;
- for (const auto *Op : Expr->operands()) {
+ for (SCEVUse Op : Expr->operands()) {
Operands.push_back(
SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(Op));
Changed |= Op != Operands.back();
diff --git a/llvm/lib/Analysis/ScalarEvolutionDivision.cpp b/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
index 52d82785f6b9c..710473191f8c6 100644
--- a/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
+++ b/llvm/lib/Analysis/ScalarEvolutionDivision.cpp
@@ -149,7 +149,7 @@ void SCEVDivision::visitAddRecExpr(const SCEVAddRecExpr *Numerator) {
}
void SCEVDivision::visitAddExpr(const SCEVAddExpr *Numerator) {
- SmallVector<const SCEV *, 2> Qs, Rs;
+ SmallVector<SCEVUse, 2> Qs, Rs;
Type *Ty = Denominator->getType();
for (const SCEV *Op : Numerator->operands()) {
@@ -175,7 +175,7 @@ void SCEVDivision::visitAddExpr(const SCEVAddExpr *Numerator) {
}
void SCEVDivision::visitMulExpr(const SCEVMulExpr *Numerator) {
- SmallVector<const SCEV *, 2> Qs;
+ SmallVector<SCEVUse, 2> Qs;
Type *Ty = Denominator->getType();
bool FoundDenominatorTerm = false;
diff --git a/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp b/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
index cfc5b84554548..679748f4338f5 100644
--- a/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
+++ b/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
@@ -49,7 +49,7 @@ NormalizeDenormalizeRewriter::visitAddRecExpr(const SCEVAddRecExpr *AR) {
SmallVector<const SCEV *, 8> Operands;
transform(AR->operands(), std::back_inserter(Operands),
- [&](const SCEV *Op) { return visit(Op); });
+ [&](SCEVUse Op) { return visit(Op.getPointer()); });
if (!Pred(AR))
return SE.getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagAnyWrap);
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index a7766d91af3f2..b67361922bd6b 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -2786,7 +2786,7 @@ void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
bool Runtime = true;
if (ST->hasLOB()) {
if (SE.hasLoopInvariantBackedgeTakenCount(L)) {
- const auto *BETC = SE.getBackedgeTakenCount(L);
+ const SCEV *BETC = SE.getBackedgeTakenCount(L);
auto *Outer = L->getOutermostLoop();
if ((L != Outer && Outer != L->getParentLoop()) ||
(L != Outer && BETC && !SE.isLoopInvariant(BETC, Outer))) {
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 305fe05c4f180..0cfb135046178 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1397,8 +1397,9 @@ createReplacement(ICmpInst *ICmp, const Loop *L, BasicBlock *ExitingBB,
// So we manually construct umin(a - 1, b - 1).
SmallVector<const SCEV *, 4> Elements;
if (auto *UMin = dyn_cast<SCEVUMinExpr>(MaxIter)) {
- for (const SCEV *Op : UMin->operands())
- Elements.push_back(SE->getMinusSCEV(Op, SE->getOne(Op->getType())));
+ for (SCEVUse Op : UMin->operands())
+ Elements.push_back(SE->getMinusSCEV(
+ Op.getPointer(), SE->getOne(Op.getPointer()->getType())));
MaxIter = SE->getUMinFromMismatchedTypes(Elements);
} else
MaxIter = SE->getMinusSCEV(MaxIter, SE->getOne(MaxIter->getType()));
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index deff79b403883..603b0c168c9a8 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -423,22 +423,16 @@ bool InductiveRangeCheck::reassociateSubLHS(
auto getExprScaledIfOverflow = [&](Instruction::BinaryOps BinOp,
const SCEV *LHS,
const SCEV *RHS) -> const SCEV * {
- const SCEV *(ScalarEvolution::*Operation)(const SCEV *, const SCEV *,
- SCEV::NoWrapFlags, unsigned);
- switch (BinOp) {
- default:
- llvm_unreachable("Unsupported binary op");
- case Instruction::Add:
- Operation = &ScalarEvolution::getAddExpr;
- break;
- case Instruction::Sub:
- Operation = &ScalarEvolution::getMinusSCEV;
- break;
- }
+ const SCEV *Result = nullptr;
if (SE.willNotOverflow(BinOp, ICmpInst::isSigned(Pred), LHS, RHS,
- cast<Instruction>(VariantLHS)))
- return (SE.*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0);
+ cast<Instruction>(VariantLHS))) {
+ if (BinOp == Instruction::Add)
+ Result = SE.getAddExpr(LHS, RHS, SCEV::FlagAnyWrap);
+ else if (BinOp == Instruction::Sub)
+ Result = SE.getMinusSCEV(LHS, RHS);
+ return Result;
+ }
// We couldn't prove that the expression does not overflow.
// Than scale it to a wider type to check overflow at runtime.
@@ -447,9 +441,13 @@ bool InductiveRangeCheck::reassociateSubLHS(
return nullptr;
auto WideTy = IntegerType::get(Ty->getContext(), Ty->getBitWidth() * 2);
- return (SE.*Operation)(SE.getSignExtendExpr(LHS, WideTy),
- SE.getSignExtendExpr(RHS, WideTy), SCEV::FlagAnyWrap,
- 0);
+ const SCEV *LHSExt = SE.getSignExtendExpr(LHS, WideTy);
+ const SCEV *RHSExt = SE.getSignExtendExpr(RHS, WideTy);
+ if (BinOp == Instruction::Add)
+ Result = SE.getAddExpr(LHSExt, RHSExt, SCEV::FlagAnyWrap);
+ else if (BinOp == Instruction::Sub)
+ Result = SE.getMinusSCEV(LHSExt, RHSExt);
+ return Result;
};
if (OffsetSubtracted)
@@ -765,7 +763,7 @@ InductiveRangeCheck::computeSafeIterationSpace(ScalarEvolution &SE,
const SCEV *Zero = SE.getZero(M->getType());
// This function returns SCEV equal to 1 if X is non-negative 0 otherwise.
- auto SCEVCheckNonNegative = [&](const SCEV *X) {
+ auto SCEVCheckNonNegative = [&](const SCEV *X) -> const SCEV * {
const Loop *L = IndVar->getLoop();
const SCEV *Zero = SE.getZero(X->getType());
const SCEV *One = SE.getOne(X->getType());
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index 6f0d8dcb05e04..b58ed17e87b8f 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -391,9 +391,10 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
BasicBlock *BB = P.InsertPt->getParent();
SCEVExpander SCEVE(*SE, "prefaddr");
- const SCEV *NextLSCEV = SE->getAddExpr(P.LSCEVAddRec, SE->getMulExpr(
- SE->getConstant(P.LSCEVAddRec->getType(), ItersAhead),
- P.LSCEVAddRec->getStepRecurrence(*SE)));
+ const SCEV *NextLSCEV = SE->getAddExpr(
+ P.LSCEVAddRec,
+ SE->getMulExpr(SE->getConstant(P.LSCEVAddRec->getType(), ItersAhead),
+ P.LSCEVAddRec->getStepRecurrence(*SE).getPointer()));
if (!SCEVE.isSafeToExpand(NextLSCEV))
continue;
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index eefd602659660..f2fb2820f288c 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1146,7 +1146,8 @@ struct LoopFuser {
const Loop *ExprL = Expr->getLoop();
SmallVector<const SCEV *, 2> Operands;
if (ExprL == &OldL) {
- append_range(Operands, Expr->operands());
+ for (SCEVUse Op : Expr->operands())
+ Operands.push_back(Op.getPointer());
return SE.getAddRecExpr(Operands, &NewL, Expr->getNoWrapFlags());
}
@@ -1156,11 +1157,11 @@ struct LoopFuser {
Valid = false;
return Expr;
}
- return visit(Expr->getStart());
+ return visit(Expr->getStart().getPointer());
}
- for (const SCEV *Op : Expr->operands())
- Operands.push_back(visit(Op));
+ for (SCEVUse Op : Expr->operands())
+ Operands.push_back(visit(Op.getPointer()));
return SE.getAddRecExpr(Operands, ExprL, Expr->getNoWrapFlags());
}
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 074cc73d53080..d6200b1909226 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -889,7 +889,7 @@ bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
return false;
}
- const SCEV *MemsetSizeSCEV = SE->getSCEV(MSI->getLength());
+ SCEVUse MemsetSizeSCEV = SE->getSCEV(MSI->getLength());
bool IsNegStride = false;
const bool IsConstantSize = isa<ConstantInt>(MSI->getLength());
@@ -928,9 +928,9 @@ bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
// Compare positive direction PointerStrideSCEV with MemsetSizeSCEV
IsNegStride = PointerStrideSCEV->isNonConstantNegative();
- const SCEV *PositiveStrideSCEV =
- IsNegStride ? SE->getNegativeSCEV(PointerStrideSCEV)
- : PointerStrideSCEV;
+ SCEVUse PositiveStrideSCEV =
+ IsNegStride ? SCEVUse(SE->getNegativeSCEV(PointerStrideSCEV))
+ : SCEVUse(PointerStrideSCEV);
LLVM_DEBUG(dbgs() << " MemsetSizeSCEV: " << *MemsetSizeSCEV << "\n"
<< " PositiveStrideSCEV: " << *PositiveStrideSCEV
<< "\n");
diff --git a/llvm/lib/Transforms/Scalar/LoopPredication.cpp b/llvm/lib/Transforms/Scalar/LoopPredication.cpp
index ea261491b7f94..81d4b20202177 100644
--- a/llvm/lib/Transforms/Scalar/LoopPredication.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPredication.cpp
@@ -1006,9 +1006,9 @@ static const SCEV *getMinAnalyzeableBackedgeTakenCount(ScalarEvolution &SE,
SmallVector<BasicBlock *, 16> ExitingBlocks;
L->getExitingBlocks(ExitingBlocks);
- SmallVector<const SCEV *, 4> ExitCounts;
+ SmallVector<SCEVUse, 4> ExitCounts;
for (BasicBlock *ExitingBB : ExitingBlocks) {
- const SCEV *ExitCount = SE.getExitCount(L, ExitingBB);
+ SCEVUse ExitCount = SE.getExitCount(L, ExitingBB);
if (isa<SCEVCouldNotCompute>(ExitCount))
continue;
assert(DT.dominates(ExitingBB, L->getLoopLatch()) &&
@@ -1018,7 +1018,10 @@ static const SCEV *getMinAnalyzeableBackedgeTakenCount(ScalarEvolution &SE,
}
if (ExitCounts.size() < 2)
return SE.getCouldNotCompute();
- return SE.getUMinFromMismatchedTypes(ExitCounts);
+ SmallVector<const SCEV *, 4> ExitCountPtrs;
+ for (SCEVUse U : ExitCounts)
+ ExitCountPtrs.push_back(U.getPointer());
+ return SE.getUMinFromMismatchedTypes(ExitCountPtrs);
}
/// This implements an analogous, but entirely distinct transform from the main
diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
index ec145f2f48bea..fda12e7f3be42 100644
--- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -225,7 +225,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
for (const auto Node : depth_first(DT)) {
BasicBlock *BB = Node->getBlock();
for (Instruction &OrigI : *BB) {
- const SCEV *OrigSCEV = nullptr;
+ SCEVUse OrigSCEV = nullptr;
if (Instruction *NewI = tryReassociate(&OrigI, OrigSCEV)) {
Changed = true;
OrigI.replaceAllUsesWith(NewI);
@@ -234,7 +234,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
DeadInsts.push_back(WeakTrackingVH(&OrigI));
// Add the rewritten instruction to SeenExprs; the original
// instruction is deleted.
- const SCEV *NewSCEV = SE->getSCEV(NewI);
+ SCEVUse NewSCEV = SE->getSCEV(NewI);
SeenExprs[NewSCEV].push_back(WeakTrackingVH(NewI));
// Ideally, NewSCEV should equal OldSCEV because tryReassociate(I)
@@ -273,7 +273,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
template <typename PredT>
Instruction *
NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
- const SCEV *&OrigSCEV) {
+ SCEVUse &OrigSCEV) {
Value *LHS = nullptr;
Value *RHS = nullptr;
@@ -292,8 +292,8 @@ NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
return nullptr;
}
-Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
- const SCEV *&OrigSCEV) {
+Instruction *NaryReassociatePass::tryReassociate(Instruction *I,
+ SCEVUse &OrigSCEV) {
if (!SE->isSCEVable(I->getType()))
return nullptr;
@@ -397,7 +397,7 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
Value *RHS, Type *IndexedType) {
// Look for GEP's closest dominator that has the same SCEV as GEP except that
// the I-th index is replaced with LHS.
- SmallVector<const SCEV *, 4> IndexExprs;
+ SmallVector<SCEVUse, 4> IndexExprs;
for (Use &Index : GEP->indices())
IndexExprs.push_back(SE->getSCEV(Index));
// Replace the I-th index with LHS.
@@ -414,8 +414,10 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
// @reassociate_gep_assume for an example of this canonicalization.
IndexExprs[I] = SE->getZeroExtendExpr(IndexExprs[I], GEPArgType);
}
- const SCEV *CandidateExpr = SE->getGEPExpr(cast<GEPOperator>(GEP),
- IndexExprs);
+ SmallVector<const SCEV *, 4> IndexPtrs;
+ for (SCEVUse U : IndexExprs)
+ IndexPtrs.push_back(U.getPointer());
+ SCEVUse CandidateExpr = SE->getGEPExpr(cast<GEPOperator>(GEP), IndexPtrs);
Value *Candidate = findClosestMatchingDominator(CandidateExpr, GEP);
if (Candidate == nullptr)
@@ -481,8 +483,8 @@ Instruction *NaryReassociatePass::tryReassociateBinaryOp(Value *LHS, Value *RHS,
if (LHS->hasOneUse() && matchTernaryOp(I, LHS, A, B)) {
// I = (A op B) op RHS
// = (A op RHS) op B or (B op RHS) op A
- const SCEV *AExpr = SE->getSCEV(A), *BExpr = SE->getSCEV(B);
- const SCEV *RHSExpr = SE->getSCEV(RHS);
+ SCEVUse AExpr = SE->getSCEV(A), BExpr = SE->getSCEV(B);
+ SCEVUse RHSExpr = SE->getSCEV(RHS);
if (BExpr != RHSExpr) {
if (auto *NewI =
tryReassociatedBinaryOp(getBinarySCEV(I, AExpr, RHSExpr), B, I))
@@ -497,7 +499,7 @@ Instruction *NaryReassociatePass::tryReassociateBinaryOp(Value *LHS, Value *RHS,
return nullptr;
}
-Instruction *NaryReassociatePass::tryReassociatedBinaryOp(const SCEV *LHSExpr,
+Instruction *NaryReassociatePass::tryReassociatedBinaryOp(SCEVUse LHSExpr,
Value *RHS,
BinaryOperator *I) {
// Look for the closest dominator LHS of I that computes LHSExpr, and replace
@@ -535,9 +537,8 @@ bool NaryReassociatePass::matchTernaryOp(BinaryOperator *I, Value *V,
return false;
}
-const SCEV *NaryReassociatePass::getBinarySCEV(BinaryOperator *I,
- const SCEV *LHS,
- const SCEV *RHS) {
+SCEVUse NaryReassociatePass::getBinarySCEV(BinaryOperator *I, SCEVUse LHS,
+ SCEVUse RHS) {
switch (I->getOpcode()) {
case Instruction::Add:
return SE->getAddExpr(LHS, RHS);
@@ -550,7 +551,7 @@ const SCEV *NaryReassociatePass::getBinarySCEV(BinaryOperator *I,
}
Instruction *
-NaryReassociatePass::findClosestMatchingDominator(const SCEV *CandidateExpr,
+NaryReassociatePass::findClosestMatchingDominator(SCEVUse CandidateExpr,
Instruction *Dominatee) {
auto Pos = SeenExprs.find(CandidateExpr);
if (Pos == SeenExprs.end())
@@ -622,12 +623,11 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
}))
return nullptr;
- auto tryCombination = [&](Value *A, const SCEV *AExpr, Value *B,
- const SCEV *BExpr, Value *C,
- const SCEV *CExpr) -> Value * {
- SmallVector<const SCEV *, 2> Ops1{BExpr, AExpr};
+ auto tryCombination = [&](Value *A, SCEVUse AExpr, Value *B, SCEVUse BExpr,
+ Value *C, SCEVUse CExpr) -> Value * {
+ SmallVector<const SCEV *, 2> Ops1{BExpr.getPointer(), AExpr.getPointer()};
const SCEVTypes SCEVType = convertToSCEVype(m_MaxMin);
- const SCEV *R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
+ SCEVUse R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
Instruction *R1MinMax = findClosestMatchingDominator(R1Expr, I);
@@ -638,7 +638,7 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
SmallVector<const SCEV *, 2> Ops2{SE->getUnknown(C),
SE->getUnknown(R1MinMax)};
- const SCEV *R2Expr = SE->getMinMaxExpr(SCEVType, Ops2);
+ SCEVUse R2Expr = SE->getMinMaxExpr(SCEVType, Ops2);
SCEVExpander Expander(*SE, "nary-reassociate");
Value *NewMinMax = Expander.expandCodeFor(R2Expr, I->getType(), I);
@@ -649,9 +649,9 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
return NewMinMax;
};
- const SCEV *AExpr = SE->getSCEV(A);
- const SCEV *BExpr = SE->getSCEV(B);
- const SCEV *RHSExpr = SE->getSCEV(RHS);
+ SCEVUse AExpr = SE->getSCEV(A);
+ SCEVUse BExpr = SE->getSCEV(B);
+ SCEVUse RHSExpr = SE->getSCEV(RHS);
if (BExpr != RHSExpr) {
// Try (A op RHS) op B
diff --git a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
index c371a9d2f6416..8672ba022b347 100644
--- a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
@@ -1086,7 +1086,7 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
if (GEP->getType()->isVectorTy())
return;
- SmallVector<const SCEV *, 4> IndexExprs;
+ SmallVector<SCEVUse, 4> IndexExprs;
for (Use &Idx : GEP->indices())
IndexExprs.push_back(SE->getSCEV(Idx));
@@ -1095,12 +1095,15 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
if (GTI.isStruct())
continue;
- const SCEV *OrigIndexExpr = IndexExprs[I - 1];
- IndexExprs[I - 1] = SE->getZero(OrigIndexExpr->getType());
+ SCEVUse OrigIndexExpr = IndexExprs[I - 1];
+ IndexExprs[I - 1] = SE->getZero(OrigIndexExpr.getPointer()->getType());
// The base of this candidate is GEP's base plus the offsets of all
// indices except this current one.
- const SCEV *BaseExpr = SE->getGEPExpr(cast<GEPOperator>(GEP), IndexExprs);
+ SmallVector<const SCEV *, 4> IndexPtrs;
+ for (SCEVUse U : IndexExprs)
+ IndexPtrs.push_back(U.getPointer());
+ SCEVUse BaseExpr = SE->getGEPExpr(cast<GEPOperator>(GEP), IndexPtrs);
Value *ArrayIdx = GEP->getOperand(I);
uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
IntegerType *PtrIdxTy = cast<IntegerType>(DL->getIndexType(GEP->getType()));
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index e57e0cf636501..066c3339bfdb1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -537,7 +537,8 @@ class SCEVAddRecForUniformityRewriter
const SCEV *NewStep =
SE.getMulExpr(Step, SE.getConstant(Ty, StepMultiplier));
const SCEV *ScaledOffset = SE.getMulExpr(Step, SE.getConstant(Ty, Offset));
- const SCEV *NewStart = SE.getAddExpr(Expr->getStart(), ScaledOffset);
+ const SCEV *NewStart =
+ SE.getAddExpr(Expr->getStart(), SCEVUse(ScaledOffset));
return SE.getAddRecExpr(NewStart, NewStep, TheLoop, SCEV::FlagAnyWrap);
}
diff --git a/polly/include/polly/Support/ScopHelper.h b/polly/include/polly/Support/ScopHelper.h
index 38b731a9f7d8d..fa36acf458823 100644
--- a/polly/include/polly/Support/ScopHelper.h
+++ b/polly/include/polly/Support/ScopHelper.h
@@ -14,6 +14,7 @@
#define POLLY_SUPPORT_IRHELPER_H
#include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/ValueHandle.h"
@@ -37,7 +38,7 @@ class Scop;
class ScopStmt;
/// Same as llvm/Analysis/ScalarEvolutionExpressions.h
-using LoopToScevMapT = llvm::DenseMap<const llvm::Loop *, const llvm::SCEV *>;
+using LoopToScevMapT = llvm::DenseMap<const llvm::Loop *, llvm::SCEVUse>;
/// Enumeration of assumptions Polly can take.
enum AssumptionKind {
diff --git a/polly/lib/Support/ScopHelper.cpp b/polly/lib/Support/ScopHelper.cpp
index fe9b8bb1ffea1..5bd4c85bb57bc 100644
--- a/polly/lib/Support/ScopHelper.cpp
+++ b/polly/lib/Support/ScopHelper.cpp
@@ -401,38 +401,38 @@ struct ScopExpander final : SCEVVisitor<ScopExpander, const SCEV *> {
return GenSE.getMulExpr(NewOps);
}
const SCEV *visitUMaxExpr(const SCEVUMaxExpr *E) {
- SmallVector<const SCEV *, 4> NewOps;
- for (const SCEV *Op : E->operands())
+ SmallVector<SCEVUse, 4> NewOps;
+ for (SCEVUse Op : E->operands())
NewOps.push_back(visit(Op));
return GenSE.getUMaxExpr(NewOps);
}
const SCEV *visitSMaxExpr(const SCEVSMaxExpr *E) {
- SmallVector<const SCEV *, 4> NewOps;
- for (const SCEV *Op : E->operands())
+ SmallVector<SCEVUse, 4> NewOps;
+ for (SCEVUse Op : E->operands())
NewOps.push_back(visit(Op));
return GenSE.getSMaxExpr(NewOps);
}
const SCEV *visitUMinExpr(const SCEVUMinExpr *E) {
- SmallVector<const SCEV *, 4> NewOps;
- for (const SCEV *Op : E->operands())
+ SmallVector<SCEVUse, 4> NewOps;
+ for (SCEVUse Op : E->operands())
NewOps.push_back(visit(Op));
return GenSE.getUMinExpr(NewOps);
}
const SCEV *visitSMinExpr(const SCEVSMinExpr *E) {
- SmallVector<const SCEV *, 4> NewOps;
- for (const SCEV *Op : E->operands())
+ SmallVector<SCEVUse, 4> NewOps;
+ for (SCEVUse Op : E->operands())
NewOps.push_back(visit(Op));
return GenSE.getSMinExpr(NewOps);
}
const SCEV *visitSequentialUMinExpr(const SCEVSequentialUMinExpr *E) {
- SmallVector<const SCEV *, 4> NewOps;
- for (const SCEV *Op : E->operands())
+ SmallVector<SCEVUse, 4> NewOps;
+ for (SCEVUse Op : E->operands())
NewOps.push_back(visit(Op));
return GenSE.getUMinExpr(NewOps, /*Sequential=*/true);
}
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *E) {
- SmallVector<const SCEV *, 4> NewOps;
- for (const SCEV *Op : E->operands())
+ SmallVector<SCEVUse, 4> NewOps;
+ for (SCEVUse Op : E->operands())
NewOps.push_back(visit(Op));
const Loop *L = E->getLoop();
>From 03983d57f8d650d69508784eb94c5906b2fdf5b2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 19 Feb 2026 18:02:05 +0000
Subject: [PATCH 2/3] [SCEV] Add canonical SCEV pointer
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index b21ea762094ec..6055869237b06 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -81,7 +81,7 @@ struct SCEVUse : public PointerIntPair<const SCEV *, 3> {
bool isCanonical() const;
- const SCEV *getCanonical() const { return getPointer(); }
+ const SCEV *getCanonical() const;
unsigned getFlags() const { return getInt(); }
@@ -196,6 +196,8 @@ class SCEV : public FoldingSetNode {
/// miscellaneous information.
unsigned short SubclassData = 0;
+ const SCEV *CanonicalSCEV;
+
public:
/// NoWrapFlags are bitfield indices into SubclassData.
///
@@ -243,7 +245,7 @@ class SCEV : public FoldingSetNode {
explicit SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
unsigned short ExpressionSize)
- : FastID(ID), SCEVType(SCEVTy), ExpressionSize(ExpressionSize) {}
+ : FastID(ID), SCEVType(SCEVTy), ExpressionSize(ExpressionSize), CanonicalSCEV(this) {}
SCEV(const SCEV &) = delete;
SCEV &operator=(const SCEV &) = delete;
@@ -286,6 +288,8 @@ class SCEV : public FoldingSetNode {
/// This method is used for debugging.
LLVM_ABI void dump() const;
+
+ LLVM_ABI const SCEV *getCanonical() const { return CanonicalSCEV; }
};
// Specialize FoldingSetTrait for SCEV to avoid needing to compute
@@ -2708,6 +2712,10 @@ template <> struct DenseMapInfo<ScalarEvolution::FoldID> {
}
};
+inline const SCEV *SCEVUse::getCanonical() const {
+ return getPointer()->getCanonical();
+}
+
} // end namespace llvm
#endif // LLVM_ANALYSIS_SCALAREVOLUTION_H
>From 7ec6c71e04562702ca2684481a0801d5c1f930eb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 19 Feb 2026 19:17:47 +0000
Subject: [PATCH 3/3] [SCEV] Rewrite to always create canonical SCEV.
---
llvm/include/llvm/Analysis/ScalarEvolution.h | 10 +++-
llvm/lib/Analysis/ScalarEvolution.cpp | 60 ++++++++++++++++++++
2 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 6055869237b06..206cedd2b2f3f 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -99,6 +99,8 @@ struct SCEVUse : public PointerIntPair<const SCEV *, 3> {
/// This method is used for debugging.
void dump() const;
+ static const SCEV *computeCanonical(ScalarEvolution &SE, const SCEV *);
+
friend hash_code hash_value(const SCEVUse &U) {
return hash_value(U.getRawPointer());
}
@@ -119,6 +121,10 @@ template <> struct PointerLikeTypeTraits<SCEVUse> {
};
template <> struct DenseMapInfo<SCEVUse> {
+ // The following should hold, but it would require T to be complete:
+ // static_assert(alignof(T) <= (1 << Log2MaxAlign),
+ // "DenseMap does not support pointer keys requiring more than "
+ // "Log2MaxAlign bits of alignment");
static constexpr uintptr_t Log2MaxAlign = 12;
static inline SCEVUse getEmptyKey() {
@@ -245,7 +251,7 @@ class SCEV : public FoldingSetNode {
explicit SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
unsigned short ExpressionSize)
- : FastID(ID), SCEVType(SCEVTy), ExpressionSize(ExpressionSize), CanonicalSCEV(this) {}
+ : FastID(ID), SCEVType(SCEVTy), ExpressionSize(ExpressionSize) {}
SCEV(const SCEV &) = delete;
SCEV &operator=(const SCEV &) = delete;
@@ -289,6 +295,8 @@ class SCEV : public FoldingSetNode {
/// This method is used for debugging.
LLVM_ABI void dump() const;
+ LLVM_ABI void computeAndSetCanonical(ScalarEvolution &SE);
+
LLVM_ABI const SCEV *getCanonical() const { return CanonicalSCEV; }
};
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 8cc118ed2e784..3be3213fb8dd3 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -258,6 +258,49 @@ static cl::opt<bool> UseContextForNoWrapFlagInference(
// SCEV class definitions
//===----------------------------------------------------------------------===//
+class SCEVDropFlags : public SCEVRewriteVisitor<SCEVDropFlags> {
+ using Base = SCEVRewriteVisitor<SCEVDropFlags>;
+
+public:
+ SCEVDropFlags(ScalarEvolution &SE) : SCEVRewriteVisitor(SE) {}
+
+ static const SCEV *rewrite(SCEVUse Scev, ScalarEvolution &SE) {
+ SCEVDropFlags Rewriter(SE);
+ return Rewriter.visit(Scev);
+ }
+
+ const SCEV *visit(const SCEV *S) {
+ if (S->getCanonical() == S)
+ return S;
+ return Base::visit(S);
+ }
+
+ const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
+ SmallVector<SCEVUse, 2> Operands;
+ bool Changed = false;
+ for (const auto Op : Expr->operands()) {
+ Operands.push_back(visit(Op));
+ Changed |= Op != Operands.back();
+ }
+ return !Changed ? Expr : SE.getAddExpr(Operands, Expr->getNoWrapFlags());
+ }
+
+ const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
+ SmallVector<SCEVUse, 2> Operands;
+ bool Changed = false;
+ for (const auto Op : Expr->operands()) {
+ Operands.push_back(visit(Op));
+ Changed |= Op != Operands.back();
+ }
+ return !Changed ? Expr : SE.getMulExpr(Operands, Expr->getNoWrapFlags());
+ }
+};
+
+void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
+
+ CanonicalSCEV = SCEVDropFlags::rewrite(this, SE);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void SCEVUse::dump() const {
print(dbgs());
@@ -494,6 +537,7 @@ const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
void *IP = nullptr;
if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
+ S->computeAndSetCanonical(*this);
UniqueSCEVs.InsertNode(S, IP);
return S;
}
@@ -519,6 +563,7 @@ const SCEV *ScalarEvolution::getVScale(Type *Ty) {
if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(SCEVAllocator), Ty);
+ S->computeAndSetCanonical(*this);
UniqueSCEVs.InsertNode(S, IP);
return S;
}
@@ -1133,6 +1178,7 @@ const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op) {
SCEV *S = new (SCEVAllocator)
SCEVPtrToIntExpr(ID.Intern(SCEVAllocator), U, IntPtrTy);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, U);
return static_cast<const SCEV *>(S);
});
@@ -1166,6 +1212,7 @@ const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
SCEV *S = new (SCEVAllocator)
SCEVPtrToAddrExpr(ID.Intern(SCEVAllocator), U, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, U);
return static_cast<const SCEV *>(S);
});
@@ -1222,6 +1269,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty,
SCEV *S =
new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Op);
return S;
}
@@ -1275,6 +1323,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty,
SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Op);
return S;
}
@@ -1645,6 +1694,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Op);
return S;
}
@@ -1929,6 +1979,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Op);
return S;
}
@@ -1985,6 +2036,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Op);
return S;
}
@@ -2191,6 +2243,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, ArrayRef<SCEVUse>(SCEVUse(Op)));
return S;
}
@@ -3047,6 +3100,7 @@ const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
S = new (SCEVAllocator)
SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Ops);
}
S->setNoWrapFlags(Flags);
@@ -3070,6 +3124,7 @@ ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
S = new (SCEVAllocator)
SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
LoopUsers[L].push_back(S);
registerUser(S, Ops);
}
@@ -3092,6 +3147,7 @@ const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Ops);
}
S->setNoWrapFlags(Flags);
@@ -3664,6 +3720,7 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
LHS, RHS);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, ArrayRef<SCEVUse>({SCEVUse(LHS), SCEVUse(RHS)}));
return S;
}
@@ -4074,6 +4131,7 @@ ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Ops);
return S;
}
@@ -4467,6 +4525,7 @@ ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
SCEVSequentialMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
registerUser(S, Ops);
return S;
}
@@ -4558,6 +4617,7 @@ const SCEV *ScalarEvolution::getUnknown(Value *V) {
FirstUnknown);
FirstUnknown = cast<SCEVUnknown>(S);
UniqueSCEVs.InsertNode(S, IP);
+ S->computeAndSetCanonical(*this);
return S;
}
More information about the llvm-commits
mailing list