[llvm] Adding the support for vectorization of loops with load based tripcount (PR #209390)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 00:03:42 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Manish Srivastava (mk-srivastava)
<details>
<summary>Changes</summary>
The changes here will extend vectorization for loops with tripcount based on load. Current LoopVectorizer is unable to vectorize the loop as SCEV returns `SCEVCouldNotCompute` for BackEdgeTakenCount and thus the loop vectorizer legality fails.
The changes will make the loops countable with the help of predicates and reusing/relying `LoopVectorizer`'s existing memory checks, costing and versioning path for rest for vectorization.
For accomplishing this, we are adding:
* `SCEVTripCountInvariantPredicate`: An assumption that the load based tripcount is invariant. This predicate lowers to no extra runtime checks of its own but the relevant checks are generated by the `vector.memcheck` alias checks.
* `collectInvariantLoadsBoundChain` in `LoopAccessAnalysis` which finds the chain of instructions computing the loop bound.
* Hook in `LoopVectorizer` which clones the `bound-chain` into the loop preheader and creates the `SCEVTripCountInvariantPredicate` predicate so that `PredicatedScalarEvolution` can use it to compute the BackedgeTakenCount` of the loop.
---
Patch is 71.95 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/209390.diff
16 Files Affected:
- (modified) llvm/include/llvm/Analysis/LoopAccessAnalysis.h (+24-1)
- (modified) llvm/include/llvm/Analysis/ScalarEvolution.h (+57-1)
- (modified) llvm/lib/Analysis/LoopAccessAnalysis.cpp (+175-4)
- (modified) llvm/lib/Analysis/ScalarEvolution.cpp (+116-10)
- (modified) llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp (+2)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+2-1)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+85-2)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll (+56)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll (+60)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll (+41)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll (+147)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll (+54)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll (+144)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll (+47)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll (+166)
- (added) llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll (+43)
``````````diff
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 392321448c895..d84fd0e03a5e9 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -26,6 +26,8 @@ namespace llvm {
class AAResults;
class DataLayout;
+class Instruction;
+class LoadInst;
class Loop;
class raw_ostream;
class TargetTransformInfo;
@@ -694,7 +696,8 @@ class LoopAccessInfo {
const TargetTransformInfo *TTI,
const TargetLibraryInfo *TLI, AAResults *AA,
DominatorTree *DT, LoopInfo *LI, AssumptionCache *AC,
- bool AllowPartial = false);
+ bool AllowPartial = false,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds = {});
/// Return true we can analyze the memory accesses in the loop and there are
/// no memory dependence cycles. Note that for dependences between loads &
@@ -714,6 +717,10 @@ class LoopAccessInfo {
/// could be analyzed.
bool hasAllowPartial() const { return AllowPartial; }
+ ArrayRef<const SCEVTripCountInvariantPredicate *> getTripCountInvariantPreds() const {
+ return TripCountInvariantPreds;
+ }
+
const RuntimePointerChecking *getRuntimePointerChecking() const {
return PtrRtChecking.get();
}
@@ -838,6 +845,10 @@ class LoopAccessInfo {
/// memory accesses could be analyzed.
bool AllowPartial;
+ /// Predicates for the loops inside the loop which are used to compute the tripcount and
+ /// which this analysis assumes to be invariant.
+ SmallVector<const SCEVTripCountInvariantPredicate *, 2> TripCountInvariantPreds;
+
unsigned NumLoads = 0;
unsigned NumStores = 0;
@@ -979,6 +990,15 @@ LLVM_ABI std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
DominatorTree *DT, AssumptionCache *AC,
std::optional<ScalarEvolution::LoopGuards> &LoopGuards);
+/// This function will detect whether the loop is uncountabel because the tripcount
+/// is based on a load.
+/// If yes, then it will build a dependency chain of instructions, and a list of loads
+/// which are used to compute the tripcount.
+LLVM_ABI bool collectInvariantLoadsBoundChain(
+ Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ SmallVectorImpl<Instruction *> &HoistedDeps,
+ SmallVectorImpl<LoadInst *> &BoundLoads);
+
class LoopAccessInfoManager {
/// The cache.
DenseMap<Loop *, std::unique_ptr<LoopAccessInfo>> LoopAccessInfoMap;
@@ -1000,6 +1020,9 @@ class LoopAccessInfoManager {
LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial = false);
+ LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds);
+
LLVM_ABI void clear();
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 0a9a4370d5e80..f39553d3013d2 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -379,7 +379,7 @@ class SCEVPredicate : public FoldingSetNode {
FoldingSetNodeIDRef FastID;
public:
- enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap };
+ enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap, P_TripCountInvariant };
protected:
SCEVPredicateKind Kind;
@@ -464,6 +464,37 @@ class LLVM_ABI SCEVComparePredicate final : public SCEVPredicate {
}
};
+/// This class extends the SCEVPredicate class to represent the assumption that the trip count of a loop is invariant.
+/// This predicate doesn't lower any numeric run-time check but solely depends on the memory alias checks generated
+/// for the loop by LoopVectorizer.
+/// NOTE: Users of this predicate make sure to verify that the memory alias checks are generated for the loop.
+class LLVM_ABI SCEVTripCountInvariantPredicate final : public SCEVPredicate {
+ const SCEV *TripCountLoad;
+ const SCEV *TripCountInvariantLoad;
+
+public:
+ SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
+ const SCEV *TripCountLoad,
+ const SCEV *TripCountInvariantLoad);
+
+ /// Implementation of the SCEVPredicate interface
+ bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override;
+ void print(raw_ostream &OS, unsigned Depth = 0) const override;
+ bool isAlwaysTrue() const override;
+
+ /// This predicate doesn't generate any runtime checks, and solely depends
+ /// upon the memory alias checks for the runtime check generation, so contributes
+ /// nothing to the check complexity
+ unsigned getComplexity() const override { return 0; }
+
+ const SCEV *getTripCountLoad() const { return TripCountLoad; }
+ const SCEV *getTripCountInvariantLoad() const { return TripCountInvariantLoad; }
+
+ static bool classof(const SCEVPredicate *P) {
+ return P->getKind() == P_TripCountInvariant;
+ }
+};
+
/// This class represents an assumption made on an AddRec expression. Given an
/// affine AddRec expression {a,+,b}, we assume that it has the nssw or nusw
/// flags (defined below) in the first X iterations of the loop, where X is a
@@ -1122,6 +1153,11 @@ class ScalarEvolution {
LLVM_ABI const SCEV *getPredicatedSymbolicMaxBackedgeTakenCount(
const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates);
+ /// To compute the backedge-taken count under the assumption of TripCountInvariant predicate.
+ LLVM_ABI const SCEV *computeBackedgeTakenCountWithTripCountInvariants(
+ const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
+ bool SymbolicMax = false);
+
/// Return true if the backedge taken count is either the value returned by
/// getConstantMaxBackedgeTakenCount or zero.
LLVM_ABI bool isBackedgeTakenCountMaxOrZero(const Loop *L);
@@ -1522,6 +1558,9 @@ class ScalarEvolution {
getWrapPredicate(const SCEVAddRecExpr *AR,
SCEVWrapPredicate::IncrementWrapFlags AddedFlags);
+ LLVM_ABI const SCEVPredicate *getTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad);
+
/// Re-writes the SCEV according to the Predicates in \p A.
LLVM_ABI const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L,
const SCEVPredicate &A);
@@ -1890,6 +1929,8 @@ class ScalarEvolution {
/// function as they are computed.
DenseMap<const Loop *, BackedgeTakenInfo> PredicatedBackedgeTakenCounts;
+ DenseMap<const Value *, const SCEV *> *ActiveTripCountInvariantSubst = nullptr;
+
/// Loops whose backedge taken counts directly use this non-constant SCEV.
DenseMap<const SCEV *, SmallPtrSet<PointerIntPair<const Loop *, 1, bool>, 4>>
BECountUsers;
@@ -2655,6 +2696,17 @@ class PredicatedScalarEvolution {
/// Adds a new predicate.
LLVM_ABI void addPredicate(const SCEVPredicate &Pred);
+ LLVM_ABI void addTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad);
+
+ LLVM_ABI bool hasTripCountInvariantPredicates() const {
+ return !TripCountInvariantPreds.empty();
+ }
+
+ LLVM_ABI ArrayRef<const SCEVTripCountInvariantPredicate *> getTripCountInvariantPredicates() const {
+ return TripCountInvariantPreds;
+ }
+
/// Adds all predicates in \p Preds.
LLVM_ABI void addPredicates(ArrayRef<const SCEVPredicate *> Preds);
@@ -2726,6 +2778,10 @@ class PredicatedScalarEvolution {
/// The symbolic backedge taken count.
const SCEV *SymbolicMaxBackedgeCount = nullptr;
+ SmallVector<const SCEVTripCountInvariantPredicate *, 2> TripCountInvariantPreds;
+
+ DenseMap<const Value *, const SCEV *> TripCountInvariantMap;
+
/// The constant max trip count for the loop.
std::optional<unsigned> SmallConstantMaxTripCount;
};
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 8d3c66c8fe321..aecb316fd162a 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -27,6 +27,7 @@
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -3175,13 +3176,168 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
SymbolicStrides[Ptr] = cast<SCEVUnknown>(StrideBase);
}
+static bool isBoundLoadSafeToLoadAtCtx(Value *Ptr, Type *Ty, Align Alignment,
+ Instruction *CtxI, DominatorTree *DT,
+ AssumptionCache *AC) {
+ const DataLayout &DL = CtxI->getDataLayout();
+ for (DomTreeNode *Node = DT->getNode(CtxI->getParent()); Node;
+ Node = Node->getIDom()) {
+ BasicBlock *BB = Node->getBlock();
+ Instruction *ScanFrom =
+ (BB == CtxI->getParent()) ? CtxI : BB->getTerminator();
+ if (isSafeToLoadUnconditionally(Ptr, Ty, Alignment, DL, ScanFrom, AC, DT)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool isSafeToHoistBoundLoad(Instruction *I,
+ const SmallPtrSetImpl<Value *> &ModifiedPtrs,
+ Instruction *CtxI, DominatorTree *DT,
+ AssumptionCache *AC) {
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (!LI->isSimple()) {
+ return false;
+ }
+ Value *Ptr = LI->getPointerOperand();
+ if (ModifiedPtrs.count(Ptr)) {
+ return false;
+ }
+ if (!isBoundLoadSafeToLoadAtCtx(Ptr, LI->getType(), LI->getAlign(), CtxI, DT,
+ AC)) {
+ return false;
+ }
+ return true;
+ }
+
+ // Limiting down the kind of instructions we can hoist to avoid any unforeseen side effects.
+ if (I->isBinaryOp() || I->isUnaryOp() || I->isCast() || isa<GetElementPtrInst>(I)) {
+ return isSafeToSpeculativelyExecute(I, CtxI, AC, DT);
+ }
+ return false;
+}
+
+LLVM_ABI bool llvm::collectInvariantLoadsBoundChain(
+ Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ SmallVectorImpl<Instruction *> &HoistedDeps,
+ SmallVectorImpl<LoadInst *> &BoundLoads) {
+ HoistedDeps.clear();
+ BoundLoads.clear();
+
+ BasicBlock *ExitingBB = L->getExitingBlock();
+ if (!ExitingBB || ExitingBB != L->getLoopLatch()) {
+ return false;
+ }
+ auto *ExitBranch = dyn_cast<CondBrInst>(ExitingBB->getTerminator());
+ if (!ExitBranch) {
+ return false;
+ }
+ auto *ExitCmp = dyn_cast<ICmpInst>(ExitBranch->getCondition());
+ if (!ExitCmp) {
+ return false;
+ }
+ PHINode *IndVar = L->getInductionVariable(*SE);
+ if (!IndVar) {
+ return false;
+ }
+ Value *StepInst = IndVar->getIncomingValueForBlock(L->getLoopLatch());
+ auto IsIVOrStep = [&](Value *V) { return V == IndVar || V == StepInst; };
+
+ Value *DynamicUpperBound = nullptr;
+ if (IsIVOrStep(ExitCmp->getOperand(0))) {
+ DynamicUpperBound = ExitCmp->getOperand(1);
+ } else if (IsIVOrStep(ExitCmp->getOperand(1))) {
+ DynamicUpperBound = ExitCmp->getOperand(0);
+ } else {
+ return false;
+ }
+
+ /// Anything written inside the loop is a "not-invariant" pointer; the bound
+ /// load cannot share a pointer with such writes.
+ /// This is to prune out cases like for (i = 0; i < *Len; ++i) { Len[i] = ..*.. }
+ SmallPtrSet<Value *, 16> ModifiedPtrs;
+ for (BasicBlock *BB : L->getBlocks()) {
+ for (Instruction &I : *BB) {
+ if (auto *SI = dyn_cast<StoreInst>(&I)) {
+ ModifiedPtrs.insert(SI->getPointerOperand());
+ }
+ }
+ }
+
+ BasicBlock *Preheader = L->getLoopPreheader();
+ if (!Preheader) {
+ return false;
+ }
+ Instruction *CtxI = Preheader->getTerminator();
+
+ // The bound must depend on at least one in-loop load (otherwise the loop
+ // would already be countable and this is unnecessary).
+ SmallVector<Instruction *, 16> Worklist;
+ SmallPtrSet<Instruction *, 16> VisitedForHoisting;
+ SmallPtrSet<Instruction *, 16> VisitedForChecking;
+
+ if (auto *I = dyn_cast<Instruction>(DynamicUpperBound)) {
+ Worklist.push_back(I);
+ }
+
+ while (!Worklist.empty()) {
+ Instruction *I = Worklist.back();
+
+ if (VisitedForHoisting.count(I)) {
+ Worklist.pop_back();
+ continue;
+ }
+
+ if (!L->contains(I->getParent())) {
+ Worklist.pop_back();
+ continue;
+ }
+
+ if (VisitedForChecking.insert(I).second) {
+ if (!isSafeToHoistBoundLoad(I, ModifiedPtrs, CtxI, DT, AC)) {
+ return false;
+ }
+
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ BoundLoads.push_back(LI);
+ }
+
+ for (Use &U : I->operands()) {
+ if (auto *OpI = dyn_cast<Instruction>(U.get())) {
+ if (L->contains(OpI)) {
+ Worklist.push_back(OpI);
+ }
+ }
+ }
+
+ continue;
+ }
+
+ Worklist.pop_back();
+
+ VisitedForHoisting.insert(I);
+
+ HoistedDeps.push_back(I);
+ }
+
+ return !BoundLoads.empty() && !HoistedDeps.empty();
+}
+
LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
const TargetTransformInfo *TTI,
const TargetLibraryInfo *TLI, AAResults *AA,
DominatorTree *DT, LoopInfo *LI,
- AssumptionCache *AC, bool AllowPartial)
+ AssumptionCache *AC, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds)
: PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
- PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial) {
+ PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial),
+ TripCountInvariantPreds(TripCountInvariantPreds) {
+
+ for (const SCEVTripCountInvariantPredicate *P : TripCountInvariantPreds) {
+ PSE->addTripCountInvariantPredicate(P->getTripCountLoad(), P->getTripCountInvariantLoad());
+ }
+
unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
if (TTI && !TTI->enableScalableVectorization())
// Scale the vector width by 2 as rough estimate to also consider
@@ -3254,16 +3410,31 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
bool AllowPartial) {
+ return getInfo(L, AllowPartial, {});
+}
+
+const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
+ bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds) {
const auto &[It, Inserted] = LoopAccessInfoMap.try_emplace(&L);
+ auto SamePreds = [&]() {
+ ArrayRef<const SCEVTripCountInvariantPredicate *> Cached =
+ It->second->getTripCountInvariantPreds();
+ if (Cached.size() != TripCountInvariantPreds.size())
+ return false;
+ return std::equal(Cached.begin(), Cached.end(), TripCountInvariantPreds.begin());
+ };
+
// We need to create the LoopAccessInfo if either we don't already have one,
// or if it was created with a different value of AllowPartial.
- if (Inserted || It->second->hasAllowPartial() != AllowPartial)
+ if (Inserted || It->second->hasAllowPartial() != AllowPartial || !SamePreds())
It->second = std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT,
- &LI, AC, AllowPartial);
+ &LI, AC, AllowPartial, TripCountInvariantPreds);
return *It->second;
}
+
void LoopAccessInfoManager::clear() {
// Collect LoopAccessInfo entries that may keep references to IR outside the
// analyzed loop or SCEVs that may have been modified or invalidated. At the
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 134be6ac097e0..a204c46947001 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8676,6 +8676,29 @@ const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
return getPredicatedBackedgeTakenInfo(L).getConstantMax(this, &Preds);
}
+const SCEV *ScalarEvolution::computeBackedgeTakenCountWithTripCountInvariants(
+ const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
+ bool SymbolicMax) {
+ if (Preds.empty())
+ return SymbolicMax ? getSymbolicMaxBackedgeTakenCount(L)
+ : getBackedgeTakenCount(L);
+
+ ValueToSCEVMapTy Subst;
+ for (const SCEVTripCountInvariantPredicate *P : Preds) {
+ const auto *V = dyn_cast<SCEVUnknown>(P->getTripCountLoad());
+ if (!V)
+ return getCouldNotCompute();
+ Subst[V->getValue()] = P->getTripCountInvariantLoad();
+ }
+
+ (void)getBackedgeTakenInfo(L);
+
+ SaveAndRestore<DenseMap<const Value *, const SCEV *> *> Active(
+ ActiveTripCountInvariantSubst, &Subst);
+ BackedgeTakenInfo BTI = computeBackedgeTakenCount(L, /*AllowPredicates=*/false);
+ return SymbolicMax ? BTI.getSymbolicMax(L, this) : BTI.getExact(L, this);
+}
+
bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
}
@@ -9464,6 +9487,15 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
+ // If the TripCountInvariant predicate is active, then replace the LHS and RHS to their proper
+ // substition values as per the substitution map.
+ if (ActiveTripCountInvariantSubst && !ActiveTripCountInvariantSubst->empty()) {
+ auto &Subst = const_cast<ValueToSCEVMapTy &>(*ActiveTripCountInvariantSubst);
+
+ LHS = SCEVParameterRewriter::rewrite(LHS, *this, Subst);
+ RHS = SCEVParameterRewriter::rewrite(RHS, *this, Subst);
+ }
+
ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsOnlyExit,
AllowPredicates);
if (EL.hasAnyInfo())
@@ -15263,6 +15295,21 @@ const SCEVPredicate *ScalarEvolution::getWrapPredicate(
return OF;
}
+const SCEVPredicate *ScalarEvolution::getTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad) {
+ FoldingSetNodeID ID;
+ ID.AddInteger(SCEVPredicate::P_TripCountInvariant);
+ ID.AddPointer(Load);
+ ID.AddPointer(InvariantLoad);
+ void *IP = nullptr;
+ if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
+ return S;
+ auto *TCI = new (SCEVAllocator)
+ SCEVTripCountInvariantPredicate(ID.Intern(SCEVAllocator), Load, InvariantLoad);
+ UniquePreds.InsertNode(TCI, IP);
+ return TCI;
+}
+
namespace {
class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
@@ -15470,6 +15517,29 @@ void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
}
+SCEVTripCountInvariantPredicate::SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
+ const SCEV *TripCountLoad,
+ const SCEV *TripCountInvariantLoad)
+ : SCEVPredicate(ID, P_TripCountInvariant), TripCountLoad(TripCountLoad), TripCountInvariantLoad(TripCountInvariantLoad) {}
+
+bool SCEVTripCountInvariantPredicate::implies(const SCEVPredicate *N,
+ ScalarEvolution &SE) const {
+ const auto *Op = dyn_cast<SCEVTripCountInvariantPredicate>(N);
+ if (!Op)
+ return false;
+ return Op->TripCountLoad == TripCountLoad && Op->TripCountInvariantLoad == TripCountInvariantLoad;
+}
+
+bool SCEVTripCountInvariantPredicate::isAlwaysTrue() const {
+ return TripCountLoad == TripCountInvariantLoad;
+}
+
+void SCEVTripCountInvariantPredicate::print(raw_ostream &OS, unsigned Depth) const {
+ OS.indent(Depth) << "Trip count invariant...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/209390
More information about the llvm-commits
mailing list