[llvm] Adding the support for vectorization of loops with load based tripcount (PR #209390)
Manish Srivastava via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 01:20:45 PDT 2026
https://github.com/mk-srivastava updated https://github.com/llvm/llvm-project/pull/209390
>From 0adc4a1633addc0dbb28fc0ce47c0a29a804643e Mon Sep 17 00:00:00 2001
From: Manish Srivastava <manish.srivastava at amd.com>
Date: Tue, 14 Jul 2026 10:46:40 +0530
Subject: [PATCH 1/5] Adding the support for vectorization of loops with load
based tripcount
---
.../llvm/Analysis/LoopAccessAnalysis.h | 25 ++-
llvm/include/llvm/Analysis/ScalarEvolution.h | 58 +++++-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 179 +++++++++++++++++-
llvm/lib/Analysis/ScalarEvolution.cpp | 126 +++++++++++-
.../Utils/ScalarEvolutionExpander.cpp | 2 +
.../Vectorize/LoopVectorizationLegality.cpp | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 87 ++++++++-
.../dynamic-bound-array-element.ll | 56 ++++++
.../dynamic-bound-chained-loads.ll | 60 ++++++
.../LoopVectorize/dynamic-bound-fcmp-exit.ll | 41 ++++
.../LoopVectorize/dynamic-bound-lambda.ll | 147 ++++++++++++++
.../dynamic-bound-loop-varying.ll | 54 ++++++
.../LoopVectorize/dynamic-bound-simple.ll | 144 ++++++++++++++
.../dynamic-bound-stored-pointer.ll | 47 +++++
.../LoopVectorize/dynamic-bound-two-loads.ll | 166 ++++++++++++++++
.../dynamic-bound-volatile-load.ll | 43 +++++
16 files changed, 1219 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 392321448c895..d84fd0e03a5e9 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -26,6 +26,8 @@ namespace llvm {
class AAResults;
class DataLayout;
+class Instruction;
+class LoadInst;
class Loop;
class raw_ostream;
class TargetTransformInfo;
@@ -694,7 +696,8 @@ class LoopAccessInfo {
const TargetTransformInfo *TTI,
const TargetLibraryInfo *TLI, AAResults *AA,
DominatorTree *DT, LoopInfo *LI, AssumptionCache *AC,
- bool AllowPartial = false);
+ bool AllowPartial = false,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds = {});
/// Return true we can analyze the memory accesses in the loop and there are
/// no memory dependence cycles. Note that for dependences between loads &
@@ -714,6 +717,10 @@ class LoopAccessInfo {
/// could be analyzed.
bool hasAllowPartial() const { return AllowPartial; }
+ ArrayRef<const SCEVTripCountInvariantPredicate *> getTripCountInvariantPreds() const {
+ return TripCountInvariantPreds;
+ }
+
const RuntimePointerChecking *getRuntimePointerChecking() const {
return PtrRtChecking.get();
}
@@ -838,6 +845,10 @@ class LoopAccessInfo {
/// memory accesses could be analyzed.
bool AllowPartial;
+ /// Predicates for the loops inside the loop which are used to compute the tripcount and
+ /// which this analysis assumes to be invariant.
+ SmallVector<const SCEVTripCountInvariantPredicate *, 2> TripCountInvariantPreds;
+
unsigned NumLoads = 0;
unsigned NumStores = 0;
@@ -979,6 +990,15 @@ LLVM_ABI std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
DominatorTree *DT, AssumptionCache *AC,
std::optional<ScalarEvolution::LoopGuards> &LoopGuards);
+/// This function will detect whether the loop is uncountabel because the tripcount
+/// is based on a load.
+/// If yes, then it will build a dependency chain of instructions, and a list of loads
+/// which are used to compute the tripcount.
+LLVM_ABI bool collectInvariantLoadsBoundChain(
+ Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ SmallVectorImpl<Instruction *> &HoistedDeps,
+ SmallVectorImpl<LoadInst *> &BoundLoads);
+
class LoopAccessInfoManager {
/// The cache.
DenseMap<Loop *, std::unique_ptr<LoopAccessInfo>> LoopAccessInfoMap;
@@ -1000,6 +1020,9 @@ class LoopAccessInfoManager {
LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial = false);
+ LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds);
+
LLVM_ABI void clear();
LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 0a9a4370d5e80..f39553d3013d2 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -379,7 +379,7 @@ class SCEVPredicate : public FoldingSetNode {
FoldingSetNodeIDRef FastID;
public:
- enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap };
+ enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap, P_TripCountInvariant };
protected:
SCEVPredicateKind Kind;
@@ -464,6 +464,37 @@ class LLVM_ABI SCEVComparePredicate final : public SCEVPredicate {
}
};
+/// This class extends the SCEVPredicate class to represent the assumption that the trip count of a loop is invariant.
+/// This predicate doesn't lower any numeric run-time check but solely depends on the memory alias checks generated
+/// for the loop by LoopVectorizer.
+/// NOTE: Users of this predicate make sure to verify that the memory alias checks are generated for the loop.
+class LLVM_ABI SCEVTripCountInvariantPredicate final : public SCEVPredicate {
+ const SCEV *TripCountLoad;
+ const SCEV *TripCountInvariantLoad;
+
+public:
+ SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
+ const SCEV *TripCountLoad,
+ const SCEV *TripCountInvariantLoad);
+
+ /// Implementation of the SCEVPredicate interface
+ bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override;
+ void print(raw_ostream &OS, unsigned Depth = 0) const override;
+ bool isAlwaysTrue() const override;
+
+ /// This predicate doesn't generate any runtime checks, and solely depends
+ /// upon the memory alias checks for the runtime check generation, so contributes
+ /// nothing to the check complexity
+ unsigned getComplexity() const override { return 0; }
+
+ const SCEV *getTripCountLoad() const { return TripCountLoad; }
+ const SCEV *getTripCountInvariantLoad() const { return TripCountInvariantLoad; }
+
+ static bool classof(const SCEVPredicate *P) {
+ return P->getKind() == P_TripCountInvariant;
+ }
+};
+
/// This class represents an assumption made on an AddRec expression. Given an
/// affine AddRec expression {a,+,b}, we assume that it has the nssw or nusw
/// flags (defined below) in the first X iterations of the loop, where X is a
@@ -1122,6 +1153,11 @@ class ScalarEvolution {
LLVM_ABI const SCEV *getPredicatedSymbolicMaxBackedgeTakenCount(
const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates);
+ /// To compute the backedge-taken count under the assumption of TripCountInvariant predicate.
+ LLVM_ABI const SCEV *computeBackedgeTakenCountWithTripCountInvariants(
+ const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
+ bool SymbolicMax = false);
+
/// Return true if the backedge taken count is either the value returned by
/// getConstantMaxBackedgeTakenCount or zero.
LLVM_ABI bool isBackedgeTakenCountMaxOrZero(const Loop *L);
@@ -1522,6 +1558,9 @@ class ScalarEvolution {
getWrapPredicate(const SCEVAddRecExpr *AR,
SCEVWrapPredicate::IncrementWrapFlags AddedFlags);
+ LLVM_ABI const SCEVPredicate *getTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad);
+
/// Re-writes the SCEV according to the Predicates in \p A.
LLVM_ABI const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L,
const SCEVPredicate &A);
@@ -1890,6 +1929,8 @@ class ScalarEvolution {
/// function as they are computed.
DenseMap<const Loop *, BackedgeTakenInfo> PredicatedBackedgeTakenCounts;
+ DenseMap<const Value *, const SCEV *> *ActiveTripCountInvariantSubst = nullptr;
+
/// Loops whose backedge taken counts directly use this non-constant SCEV.
DenseMap<const SCEV *, SmallPtrSet<PointerIntPair<const Loop *, 1, bool>, 4>>
BECountUsers;
@@ -2655,6 +2696,17 @@ class PredicatedScalarEvolution {
/// Adds a new predicate.
LLVM_ABI void addPredicate(const SCEVPredicate &Pred);
+ LLVM_ABI void addTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad);
+
+ LLVM_ABI bool hasTripCountInvariantPredicates() const {
+ return !TripCountInvariantPreds.empty();
+ }
+
+ LLVM_ABI ArrayRef<const SCEVTripCountInvariantPredicate *> getTripCountInvariantPredicates() const {
+ return TripCountInvariantPreds;
+ }
+
/// Adds all predicates in \p Preds.
LLVM_ABI void addPredicates(ArrayRef<const SCEVPredicate *> Preds);
@@ -2726,6 +2778,10 @@ class PredicatedScalarEvolution {
/// The symbolic backedge taken count.
const SCEV *SymbolicMaxBackedgeCount = nullptr;
+ SmallVector<const SCEVTripCountInvariantPredicate *, 2> TripCountInvariantPreds;
+
+ DenseMap<const Value *, const SCEV *> TripCountInvariantMap;
+
/// The constant max trip count for the loop.
std::optional<unsigned> SmallConstantMaxTripCount;
};
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 8d3c66c8fe321..aecb316fd162a 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -27,6 +27,7 @@
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -3175,13 +3176,168 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
SymbolicStrides[Ptr] = cast<SCEVUnknown>(StrideBase);
}
+static bool isBoundLoadSafeToLoadAtCtx(Value *Ptr, Type *Ty, Align Alignment,
+ Instruction *CtxI, DominatorTree *DT,
+ AssumptionCache *AC) {
+ const DataLayout &DL = CtxI->getDataLayout();
+ for (DomTreeNode *Node = DT->getNode(CtxI->getParent()); Node;
+ Node = Node->getIDom()) {
+ BasicBlock *BB = Node->getBlock();
+ Instruction *ScanFrom =
+ (BB == CtxI->getParent()) ? CtxI : BB->getTerminator();
+ if (isSafeToLoadUnconditionally(Ptr, Ty, Alignment, DL, ScanFrom, AC, DT)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool isSafeToHoistBoundLoad(Instruction *I,
+ const SmallPtrSetImpl<Value *> &ModifiedPtrs,
+ Instruction *CtxI, DominatorTree *DT,
+ AssumptionCache *AC) {
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (!LI->isSimple()) {
+ return false;
+ }
+ Value *Ptr = LI->getPointerOperand();
+ if (ModifiedPtrs.count(Ptr)) {
+ return false;
+ }
+ if (!isBoundLoadSafeToLoadAtCtx(Ptr, LI->getType(), LI->getAlign(), CtxI, DT,
+ AC)) {
+ return false;
+ }
+ return true;
+ }
+
+ // Limiting down the kind of instructions we can hoist to avoid any unforeseen side effects.
+ if (I->isBinaryOp() || I->isUnaryOp() || I->isCast() || isa<GetElementPtrInst>(I)) {
+ return isSafeToSpeculativelyExecute(I, CtxI, AC, DT);
+ }
+ return false;
+}
+
+LLVM_ABI bool llvm::collectInvariantLoadsBoundChain(
+ Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ SmallVectorImpl<Instruction *> &HoistedDeps,
+ SmallVectorImpl<LoadInst *> &BoundLoads) {
+ HoistedDeps.clear();
+ BoundLoads.clear();
+
+ BasicBlock *ExitingBB = L->getExitingBlock();
+ if (!ExitingBB || ExitingBB != L->getLoopLatch()) {
+ return false;
+ }
+ auto *ExitBranch = dyn_cast<CondBrInst>(ExitingBB->getTerminator());
+ if (!ExitBranch) {
+ return false;
+ }
+ auto *ExitCmp = dyn_cast<ICmpInst>(ExitBranch->getCondition());
+ if (!ExitCmp) {
+ return false;
+ }
+ PHINode *IndVar = L->getInductionVariable(*SE);
+ if (!IndVar) {
+ return false;
+ }
+ Value *StepInst = IndVar->getIncomingValueForBlock(L->getLoopLatch());
+ auto IsIVOrStep = [&](Value *V) { return V == IndVar || V == StepInst; };
+
+ Value *DynamicUpperBound = nullptr;
+ if (IsIVOrStep(ExitCmp->getOperand(0))) {
+ DynamicUpperBound = ExitCmp->getOperand(1);
+ } else if (IsIVOrStep(ExitCmp->getOperand(1))) {
+ DynamicUpperBound = ExitCmp->getOperand(0);
+ } else {
+ return false;
+ }
+
+ /// Anything written inside the loop is a "not-invariant" pointer; the bound
+ /// load cannot share a pointer with such writes.
+ /// This is to prune out cases like for (i = 0; i < *Len; ++i) { Len[i] = ..*.. }
+ SmallPtrSet<Value *, 16> ModifiedPtrs;
+ for (BasicBlock *BB : L->getBlocks()) {
+ for (Instruction &I : *BB) {
+ if (auto *SI = dyn_cast<StoreInst>(&I)) {
+ ModifiedPtrs.insert(SI->getPointerOperand());
+ }
+ }
+ }
+
+ BasicBlock *Preheader = L->getLoopPreheader();
+ if (!Preheader) {
+ return false;
+ }
+ Instruction *CtxI = Preheader->getTerminator();
+
+ // The bound must depend on at least one in-loop load (otherwise the loop
+ // would already be countable and this is unnecessary).
+ SmallVector<Instruction *, 16> Worklist;
+ SmallPtrSet<Instruction *, 16> VisitedForHoisting;
+ SmallPtrSet<Instruction *, 16> VisitedForChecking;
+
+ if (auto *I = dyn_cast<Instruction>(DynamicUpperBound)) {
+ Worklist.push_back(I);
+ }
+
+ while (!Worklist.empty()) {
+ Instruction *I = Worklist.back();
+
+ if (VisitedForHoisting.count(I)) {
+ Worklist.pop_back();
+ continue;
+ }
+
+ if (!L->contains(I->getParent())) {
+ Worklist.pop_back();
+ continue;
+ }
+
+ if (VisitedForChecking.insert(I).second) {
+ if (!isSafeToHoistBoundLoad(I, ModifiedPtrs, CtxI, DT, AC)) {
+ return false;
+ }
+
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ BoundLoads.push_back(LI);
+ }
+
+ for (Use &U : I->operands()) {
+ if (auto *OpI = dyn_cast<Instruction>(U.get())) {
+ if (L->contains(OpI)) {
+ Worklist.push_back(OpI);
+ }
+ }
+ }
+
+ continue;
+ }
+
+ Worklist.pop_back();
+
+ VisitedForHoisting.insert(I);
+
+ HoistedDeps.push_back(I);
+ }
+
+ return !BoundLoads.empty() && !HoistedDeps.empty();
+}
+
LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
const TargetTransformInfo *TTI,
const TargetLibraryInfo *TLI, AAResults *AA,
DominatorTree *DT, LoopInfo *LI,
- AssumptionCache *AC, bool AllowPartial)
+ AssumptionCache *AC, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds)
: PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
- PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial) {
+ PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial),
+ TripCountInvariantPreds(TripCountInvariantPreds) {
+
+ for (const SCEVTripCountInvariantPredicate *P : TripCountInvariantPreds) {
+ PSE->addTripCountInvariantPredicate(P->getTripCountLoad(), P->getTripCountInvariantLoad());
+ }
+
unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
if (TTI && !TTI->enableScalableVectorization())
// Scale the vector width by 2 as rough estimate to also consider
@@ -3254,16 +3410,31 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
bool AllowPartial) {
+ return getInfo(L, AllowPartial, {});
+}
+
+const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
+ bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds) {
const auto &[It, Inserted] = LoopAccessInfoMap.try_emplace(&L);
+ auto SamePreds = [&]() {
+ ArrayRef<const SCEVTripCountInvariantPredicate *> Cached =
+ It->second->getTripCountInvariantPreds();
+ if (Cached.size() != TripCountInvariantPreds.size())
+ return false;
+ return std::equal(Cached.begin(), Cached.end(), TripCountInvariantPreds.begin());
+ };
+
// We need to create the LoopAccessInfo if either we don't already have one,
// or if it was created with a different value of AllowPartial.
- if (Inserted || It->second->hasAllowPartial() != AllowPartial)
+ if (Inserted || It->second->hasAllowPartial() != AllowPartial || !SamePreds())
It->second = std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT,
- &LI, AC, AllowPartial);
+ &LI, AC, AllowPartial, TripCountInvariantPreds);
return *It->second;
}
+
void LoopAccessInfoManager::clear() {
// Collect LoopAccessInfo entries that may keep references to IR outside the
// analyzed loop or SCEVs that may have been modified or invalidated. At the
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 134be6ac097e0..a204c46947001 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8676,6 +8676,29 @@ const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
return getPredicatedBackedgeTakenInfo(L).getConstantMax(this, &Preds);
}
+const SCEV *ScalarEvolution::computeBackedgeTakenCountWithTripCountInvariants(
+ const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
+ bool SymbolicMax) {
+ if (Preds.empty())
+ return SymbolicMax ? getSymbolicMaxBackedgeTakenCount(L)
+ : getBackedgeTakenCount(L);
+
+ ValueToSCEVMapTy Subst;
+ for (const SCEVTripCountInvariantPredicate *P : Preds) {
+ const auto *V = dyn_cast<SCEVUnknown>(P->getTripCountLoad());
+ if (!V)
+ return getCouldNotCompute();
+ Subst[V->getValue()] = P->getTripCountInvariantLoad();
+ }
+
+ (void)getBackedgeTakenInfo(L);
+
+ SaveAndRestore<DenseMap<const Value *, const SCEV *> *> Active(
+ ActiveTripCountInvariantSubst, &Subst);
+ BackedgeTakenInfo BTI = computeBackedgeTakenCount(L, /*AllowPredicates=*/false);
+ return SymbolicMax ? BTI.getSymbolicMax(L, this) : BTI.getExact(L, this);
+}
+
bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
}
@@ -9464,6 +9487,15 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
+ // If the TripCountInvariant predicate is active, then replace the LHS and RHS to their proper
+ // substition values as per the substitution map.
+ if (ActiveTripCountInvariantSubst && !ActiveTripCountInvariantSubst->empty()) {
+ auto &Subst = const_cast<ValueToSCEVMapTy &>(*ActiveTripCountInvariantSubst);
+
+ LHS = SCEVParameterRewriter::rewrite(LHS, *this, Subst);
+ RHS = SCEVParameterRewriter::rewrite(RHS, *this, Subst);
+ }
+
ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsOnlyExit,
AllowPredicates);
if (EL.hasAnyInfo())
@@ -15263,6 +15295,21 @@ const SCEVPredicate *ScalarEvolution::getWrapPredicate(
return OF;
}
+const SCEVPredicate *ScalarEvolution::getTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad) {
+ FoldingSetNodeID ID;
+ ID.AddInteger(SCEVPredicate::P_TripCountInvariant);
+ ID.AddPointer(Load);
+ ID.AddPointer(InvariantLoad);
+ void *IP = nullptr;
+ if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
+ return S;
+ auto *TCI = new (SCEVAllocator)
+ SCEVTripCountInvariantPredicate(ID.Intern(SCEVAllocator), Load, InvariantLoad);
+ UniquePreds.InsertNode(TCI, IP);
+ return TCI;
+}
+
namespace {
class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
@@ -15470,6 +15517,29 @@ void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
}
+SCEVTripCountInvariantPredicate::SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
+ const SCEV *TripCountLoad,
+ const SCEV *TripCountInvariantLoad)
+ : SCEVPredicate(ID, P_TripCountInvariant), TripCountLoad(TripCountLoad), TripCountInvariantLoad(TripCountInvariantLoad) {}
+
+bool SCEVTripCountInvariantPredicate::implies(const SCEVPredicate *N,
+ ScalarEvolution &SE) const {
+ const auto *Op = dyn_cast<SCEVTripCountInvariantPredicate>(N);
+ if (!Op)
+ return false;
+ return Op->TripCountLoad == TripCountLoad && Op->TripCountInvariantLoad == TripCountInvariantLoad;
+}
+
+bool SCEVTripCountInvariantPredicate::isAlwaysTrue() const {
+ return TripCountLoad == TripCountInvariantLoad;
+}
+
+void SCEVTripCountInvariantPredicate::print(raw_ostream &OS, unsigned Depth) const {
+ OS.indent(Depth) << "Trip count invariant predicate: " << *TripCountLoad
+ << " is invariant (== " << *TripCountInvariantLoad << ")\n"
+ << "\n";
+}
+
SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
const SCEVAddRecExpr *AR,
IncrementWrapFlags Flags)
@@ -15684,6 +15754,11 @@ const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
Expr = Entry.second;
const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds);
+
+ if (!TripCountInvariantMap.empty()) {
+ NewSCEV = SCEVParameterRewriter::rewrite(NewSCEV, SE, TripCountInvariantMap);
+ }
+
Entry = {Generation, NewSCEV};
return NewSCEV;
@@ -15691,21 +15766,33 @@ const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
if (!BackedgeCount) {
- SmallVector<const SCEVPredicate *, 4> Preds;
- BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
- for (const auto *P : Preds)
- addPredicate(*P);
+ if (!TripCountInvariantPreds.empty()) {
+ BackedgeCount = SE.computeBackedgeTakenCountWithTripCountInvariants(&L,
+ TripCountInvariantPreds, /*SymbolicMax=*/false);
+ }
+ else {
+ SmallVector<const SCEVPredicate *, 4> Preds;
+ BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
+ for (const auto *P : Preds)
+ addPredicate(*P);
+ }
}
return BackedgeCount;
}
const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
if (!SymbolicMaxBackedgeCount) {
- SmallVector<const SCEVPredicate *, 4> Preds;
- SymbolicMaxBackedgeCount =
- SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
- for (const auto *P : Preds)
- addPredicate(*P);
+ if (!TripCountInvariantPreds.empty()) {
+ SymbolicMaxBackedgeCount = SE.computeBackedgeTakenCountWithTripCountInvariants(&L,
+ TripCountInvariantPreds, /*SymbolicMax=*/true);
+ }
+ else {
+ SmallVector<const SCEVPredicate *, 4> Preds;
+ SymbolicMaxBackedgeCount =
+ SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
+ for (const auto *P : Preds)
+ addPredicate(*P);
+ }
}
return SymbolicMaxBackedgeCount;
}
@@ -15730,6 +15817,23 @@ void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
updateGeneration();
}
+void PredicatedScalarEvolution::addTripCountInvariantPredicate(const SCEV *Load, const SCEV *InvariantLoad) {
+ const auto *V = dyn_cast<SCEVUnknown>(Load);
+ assert(V && "mem-invariant Load must be a SCEVUnknown");
+
+ const auto *P = cast<SCEVTripCountInvariantPredicate>(
+ SE.getTripCountInvariantPredicate(Load, InvariantLoad));
+
+ if (TripCountInvariantMap.insert({V->getValue(), InvariantLoad}).second) {
+ TripCountInvariantPreds.push_back(P);
+ BackedgeCount = nullptr;
+ SymbolicMaxBackedgeCount = nullptr;
+ SmallConstantMaxTripCount.reset();
+ }
+
+ addPredicate(*P);
+}
+
void PredicatedScalarEvolution::addPredicates(
ArrayRef<const SCEVPredicate *> Preds) {
for (const SCEVPredicate *P : Preds)
@@ -15787,7 +15891,9 @@ PredicatedScalarEvolution::PredicatedScalarEvolution(
: RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
Preds(std::make_unique<SCEVUnionPredicate>(Init.Preds->getPredicates(),
SE)),
- Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
+ Generation(Init.Generation), BackedgeCount(Init.BackedgeCount),
+ TripCountInvariantPreds(Init.TripCountInvariantPreds),
+ TripCountInvariantMap(Init.TripCountInvariantMap) {}
void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
// For each block.
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 125571b18dddc..f46eb9aff16e3 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2211,6 +2211,8 @@ Value *SCEVExpander::expandCodeForPredicate(const SCEVPredicate *Pred,
auto *AddRecPred = cast<SCEVWrapPredicate>(Pred);
return expandWrapPredicate(AddRecPred, IP);
}
+ case SCEVPredicate::P_TripCountInvariant:
+ return ConstantInt::getFalse(IP->getContext());
}
llvm_unreachable("Unknown SCEV predicate type");
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 8d875b2b6e492..94e101297f68a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1158,7 +1158,8 @@ bool LoopVectorizationLegality::canVectorizeIndirectUnsafeDependences() {
}
bool LoopVectorizationLegality::canVectorizeMemory() {
- LAI = &LAIs.getInfo(*TheLoop);
+ LAI = &LAIs.getInfo(*TheLoop, false,
+ PSE.getTripCountInvariantPredicates());
const OptimizationRemarkAnalysis *LAR = LAI->getReport();
if (LAR) {
ORE->emit([&]() {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1929f52ae7d95..7dbd53a83904f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -412,6 +412,12 @@ static cl::opt<bool> EnableEarlyExitVectorization(
cl::desc(
"Enable vectorization of early exit loops with uncountable exits."));
+static cl::opt<bool> EnableVectorizeLoadsAsBound(
+ "enable-vectorize-loads-as-bound", cl::init(false), cl::Hidden,
+ cl::desc("Enable vectorization of loops whose trip count is uncountable "
+ "only because their upper bound is a load with a loop-invariant "
+ "address (example: for (i = 0; i < *Len; ++i))."));
+
static cl::opt<bool> EnableEarlyExitVectorizationWithSideEffects(
"enable-early-exit-vectorization-with-side-effects", cl::init(false),
cl::Hidden,
@@ -1630,16 +1636,29 @@ class GeneratedRTChecks {
BasicBlock *LoopHeader = L->getHeader();
BasicBlock *Preheader = L->getLoopPreheader();
+ SmallVector<const SCEVPredicate *, 4> SCEVPredsVec;
+ if (const auto *U = dyn_cast<SCEVUnionPredicate>(&UnionPred)) {
+ for (const SCEVPredicate *P : U->getPredicates()) {
+ if (!isa<SCEVTripCountInvariantPredicate>(P)) {
+ SCEVPredsVec.push_back(P);
+ }
+ }
+ }
+ else if (!isa<SCEVTripCountInvariantPredicate>(&UnionPred)) {
+ SCEVPredsVec.push_back(&UnionPred);
+ }
+ SCEVUnionPredicate FilteredPred(SCEVPredsVec, *PSE.getSE());
+
// Use SplitBlock to create blocks for SCEV & memory runtime checks to
// ensure the blocks are properly added to LoopInfo & DominatorTree. Those
// may be used by SCEVExpander. The blocks will be un-linked from their
// predecessors and removed from LI & DT at the end of the function.
- if (!UnionPred.isAlwaysTrue()) {
+ if (!FilteredPred.isAlwaysTrue()) {
SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
nullptr, "vector.scevcheck");
SCEVCheckCond = SCEVExp.expandCodeForPredicate(
- &UnionPred, SCEVCheckBlock->getTerminator());
+ &FilteredPred, SCEVCheckBlock->getTerminator());
if (isa<Constant>(SCEVCheckCond)) {
// Clean up directly after expanding the predicate to a constant, to
// avoid further expansions re-using anything left over from SCEVExp.
@@ -7859,6 +7878,66 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
Phi.eraseFromParent();
}
+bool EnableLoadBoundVectorization(Loop *L, PredicatedScalarEvolution &PSE,
+ ScalarEvolution *SE, DominatorTree *DT,
+ AssumptionCache *AC) {
+ if (!L->isInnermost() || !L->isLoopSimplifyForm() ||
+ L->getNumBackEdges() != 1 || !L->getUniqueExitBlock()) {
+ return false;
+ }
+
+ if (!isa<SCEVCouldNotCompute>(SE->getBackedgeTakenCount(L))) {
+ return false;
+ }
+
+ SmallVector<Instruction *, 16> HoistedDeps;
+ SmallVector<LoadInst *, 4> BoundLoads;
+ if (!collectInvariantLoadsBoundChain(L, SE, DT, AC, HoistedDeps, BoundLoads)) {
+ return false;
+ }
+
+ BasicBlock *Preheader = L->getLoopPreheader();
+ Instruction *InsertPt = Preheader->getTerminator();
+
+ DenseMap<Value *, Value *> CloneMap;
+ for (Instruction *I : HoistedDeps) {
+ Instruction *Clone = I->clone();
+ Clone->setName(I->getName() + ".bound.pre");
+ for (Use &U : Clone->operands()) {
+ if (auto *OpI = dyn_cast<Instruction>(U.get())) {
+ if (L->contains(OpI)) {
+ U.set(CloneMap[OpI]);
+ }
+ }
+ }
+ Clone->insertBefore(InsertPt->getIterator());
+ CloneMap[I] = Clone;
+ }
+
+ bool Added = false;
+ for (LoadInst *LI : BoundLoads) {
+ auto It = CloneMap.find(LI);
+ if (It == CloneMap.end()) {
+ continue;
+ }
+ const SCEV *LoadSCEV = SE->getSCEV(LI);
+ const SCEV *InvSCEV = SE->getSCEV(It->second);
+ if (!isa<SCEVUnknown>(LoadSCEV)) {
+ continue;
+ }
+ PSE.addTripCountInvariantPredicate(LoadSCEV, InvSCEV);
+ Added = true;
+ }
+ if (!Added) {
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "LV: Assuming in-loop bound load(s) invariant to make '"
+ << L->getHeader()->getName()
+ << "' countable (discharged by memory alias check).\n");
+ return true;
+}
+
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -7898,6 +7977,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
PredicatedScalarEvolution PSE(*SE, *L);
+ if (EnableVectorizeLoadsAsBound && L->isInnermost()) {
+ EnableLoadBoundVectorization(L, PSE, SE, DT, AC);
+ }
+
// Query this against the original loop and save it here because the profile
// of the original loop header may change as the transformation happens.
bool OptForSize = llvm::shouldOptimizeForSize(
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
new file mode 100644
index 0000000000000..7d8db5c67f519
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Negative test: Loop bound loaded from an array element A[2], where A is also
+; written inside the loop. The load and store use different indices, so
+; pointer identity does not flag a conflict but then LoopAccessAnalysis
+; finds an unresolvable (same-base) dependence between the A[2] read
+; and the A[i] write and refuses to vectorize
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < A[2]; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+
+define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+entry:
+ %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
+ %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %cmp11 = icmp sgt i32 %0, 0
+ br i1 %cmp11, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %arrayidx3 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
new file mode 100644
index 0000000000000..b9a7ad48547bf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Negative test: the upper bound is **PtrLen (an indirect load).
+; LoopAccessAnalysis cannot compute SCEV bounds for the indirect
+; load (load i32, ptr %ptr) because %ptr itself is loaded inside the loop.
+; The loop will be rejected.
+;
+; C source:
+; void foo(int *A, int *B, int *C, int **PtrLen) {
+; for (int i = 0; i < **PtrLen; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %PtrLen) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+; CHECK-NOT: .bound.pre
+entry:
+ %ptr0 = load ptr, ptr %PtrLen, align 8, !tbaa !5
+ %val0 = load i32, ptr %ptr0, align 4, !tbaa !10
+ %cmp9 = icmp sgt i32 %val0, 0
+ br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !10
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !10
+ %add = add nsw i32 %2, %1
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx4, align 4, !tbaa !10
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %ptr = load ptr, ptr %PtrLen, align 8, !tbaa !5
+ %val = load i32, ptr %ptr, align 4, !tbaa !10
+ %3 = sext i32 %val to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %3
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !12
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 8}
+!6 = !{!7, i64 8, !"p1 int"}
+!7 = !{!8, i64 8, !"any pointer"}
+!8 = !{!9, i64 1, !"omnipotent char"}
+!9 = !{!"Simple C++ TBAA"}
+!10 = !{!11, !11, i64 0, i64 4}
+!11 = !{!8, i64 4, !"int"}
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.mustprogress"}
+!14 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
new file mode 100644
index 0000000000000..d10ea005bf350
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Negative test: the exit condition is an fcmp (not icmp).
+; SCEV cannot compute the trip count of a float-bounded loop, so the
+; pass enters the dynamic-bound path. It then rejects because the
+; branch condition is an FCmpInst, not an ICmpInst.
+;
+; void foo(float *A, float *B, float *Limit) {
+; for (float x = 0.0f; x < *Limit; x += 1.0f)
+; A[(int)x] = B[(int)x];
+; }
+
+define dso_local void @foo(ptr noundef %A, ptr noundef readonly %B, ptr noundef readonly %Limit) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+; CHECK-NOT: .bound.pre
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %fiv = phi float [ 0.0, %entry ], [ %fiv.next, %for.body ]
+ %gep.b = getelementptr inbounds i32, ptr %B, i64 %iv
+ %b = load i32, ptr %gep.b, align 4
+ %gep.a = getelementptr inbounds i32, ptr %A, i64 %iv
+ store i32 %b, ptr %gep.a, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %fiv.next = fadd float %fiv, 1.0
+ %lim = load float, ptr %Limit, align 4
+ %cmp = fcmp olt float %fiv.next, %lim
+ br i1 %cmp, label %for.body, label %for.exit, !llvm.loop !0
+
+for.exit:
+ ret void
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
new file mode 100644
index 0000000000000..e43a7bcb15738
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Lambda with loop bound loaded from pointer (*len).
+;
+; void foo(int *A, int *B, int *C, int *D, int *len) {
+; auto func = [&]() {
+; for (int i = 0; i < *len; i++)
+; A[i] = B[i] * C[i];
+; };
+; func();
+; }
+;
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %D, ptr noundef readonly captures(none) %len) #0 {
+;
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[D:%.*]], ptr noundef readonly captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP10_I:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP10_I]], label %[[FOR_BODY_I_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[FOR_BODY_I_PREHEADER]]:
+; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[LEN]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND05:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND16:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT7:%.*]] = and i1 [[BOUND05]], [[BOUND16]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT7]]
+; CHECK-NEXT: [[BOUND08:%.*]] = icmp ult ptr [[A]], [[SCEVGEP4]]
+; CHECK-NEXT: [[BOUND19:%.*]] = icmp ult ptr [[LEN]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]]
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[TBAA0]], !alias.scope [[META4:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD13]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_I_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY_I:.*]]
+; CHECK: [[FOR_BODY_I]]:
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_I:%.*]], %[[FOR_BODY_I]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX_I]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX3_I:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX3_I]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[MUL_I:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[ARRAYIDX5_I:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: store i32 [[MUL_I]], ptr [[ARRAYIDX5_I]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nuw nsw i64 [[INDVARS_IV_I]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT: [[CMP_I:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_I]], [[TMP10]]
+; CHECK-NEXT: br i1 [[CMP_I]], label %[[FOR_BODY_I]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %0 = load i32, ptr %len, align 4, !tbaa !5
+ %cmp10.i = icmp sgt i32 %0, 0
+ br i1 %cmp10.i, label %for.body.i, label %exit
+
+for.body.i:
+ %indvars.iv.i = phi i64 [ %indvars.iv.next.i, %for.body.i ], [ 0, %entry ]
+ %arrayidx.i = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv.i
+ %1 = load i32, ptr %arrayidx.i, align 4, !tbaa !5
+ %arrayidx3.i = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv.i
+ %2 = load i32, ptr %arrayidx3.i, align 4, !tbaa !5
+ %mul.i = mul nsw i32 %2, %1
+ %arrayidx5.i = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv.i
+ store i32 %mul.i, ptr %arrayidx5.i, align 4, !tbaa !5
+ %indvars.iv.next.i = add nuw nsw i64 %indvars.iv.i, 1
+ %3 = load i32, ptr %len, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp.i = icmp slt i64 %indvars.iv.next.i, %4
+ br i1 %cmp.i, label %for.body.i, label %exit, !llvm.loop !9
+
+exit:
+ ret void
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
+; CHECK: [[META11]] = !{[[META5]], [[META8]], [[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
+; CHECK: [[META14]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META15]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META14]], [[META15]], [[META16]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
new file mode 100644
index 0000000000000..5ad94b333f144
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Negative test: the bound is Len[i], which depends on the
+; induction variable hence not invariant. The dependency chain traces back to a loop header
+; PHI, so the loop will be rejected.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < Len[i]; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+; CHECK-NOT: .bound.pre
+entry:
+ %0 = load i32, ptr %Len, align 4, !tbaa !5
+ %cmp12 = icmp sgt i32 %0, 0
+ br i1 %cmp12, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx4, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx6 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx6, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %arrayidx = getelementptr inbounds nuw i32, ptr %Len, i64 %indvars.iv.next
+ %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
new file mode 100644
index 0000000000000..c6e6d344e2fec
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Simple loop with loop bound loaded from a pointer (*Len).
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < *Len; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len) #0 {
+;
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readonly captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
+; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[LEN]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND05:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND16:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT7:%.*]] = and i1 [[BOUND05]], [[BOUND16]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT7]]
+; CHECK-NEXT: [[BOUND08:%.*]] = icmp ult ptr [[A]], [[SCEVGEP4]]
+; CHECK-NEXT: [[BOUND19:%.*]] = icmp ult ptr [[LEN]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]]
+; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT10]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[TBAA0]], !alias.scope [[META4:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD13]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %Len, align 4, !tbaa !5
+ %cmp9 = icmp sgt i32 %0, 0
+ br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %add = add nsw i32 %2, %1
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx4, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %3 = load i32, ptr %Len, align 4, !tbaa !5
+ %4 = sext i32 %3 to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
+; CHECK: [[META11]] = !{[[META5]], [[META8]], [[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
+; CHECK: [[META14]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META15]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META14]], [[META15]], [[META16]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
new file mode 100644
index 0000000000000..ecb087261e3af
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; The bound pointer %Len is also written to inside the loop body with a computed value,
+; so the bound changes each iteration and cannot be hoisted. The ModifiedPtrs check detects that the same
+; pointer is used as both a store target and the bound load's address.
+;
+; void foo(int *A, int *B, int *C, int *Len) {
+; for (int i = 0; i < *Len; i++) {
+; A[i] = B[i] + C[i];
+; *Len = B[i] + C[i]; // overwrite bound with computed value
+; }
+; }
+
+define dso_local void @foo(ptr noundef %A, ptr noundef readonly %B, ptr noundef readonly %C, ptr noundef %Len) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+; CHECK-NOT: .bound.pre
+entry:
+ %len0 = load i32, ptr %Len, align 4
+ %cmp0 = icmp sgt i32 %len0, 0
+ br i1 %cmp0, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx2, align 4
+ %add = add nsw i32 %1, %0
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx4, align 4
+ store i32 %add, ptr %Len, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %len = load i32, ptr %Len, align 4
+ %ext = sext i32 %len to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %ext
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
new file mode 100644
index 0000000000000..240f4f0143e95
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Loop bound is *Len1 + *Len2. Two loads and an add are hoisted to the preheader for the dynamic bound.
+;
+; void foo(int *A, int *B, int *C, int *Len1, int *Len2) {
+; for (int i = 0; i < *Len1 + *Len2; i++)
+; A[i] = B[i] + C[i];
+; }
+;
+
+define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len1, ptr noundef readonly captures(none) %Len2) #0 {
+;
+; CHECK-LABEL: define dso_local void @foo(
+; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readonly captures(none) [[LEN1:%.*]], ptr noundef readonly captures(none) [[LEN2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN1]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[LEN2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD10:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[ADD10]], 0
+; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN1]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = load i32, ptr [[LEN2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD_BOUND_PRE:%.*]] = add nsw i32 [[DOTBOUND_PRE1]], [[DOTBOUND_PRE]]
+; CHECK-NEXT: [[DOTBOUND_PRE2:%.*]] = sext i32 [[ADD_BOUND_PRE]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[DOTBOUND_PRE1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[SMAX18:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX18]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[DOTBOUND_PRE1]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP7]], i64 1)
+; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP8]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[LEN1]], i64 4
+; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[LEN2]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND07:%.*]] = icmp ult ptr [[A]], [[SCEVGEP4]]
+; CHECK-NEXT: [[BOUND18:%.*]] = icmp ult ptr [[C]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT9:%.*]] = and i1 [[BOUND07]], [[BOUND18]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT9]]
+; CHECK-NEXT: [[BOUND010:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
+; CHECK-NEXT: [[BOUND111:%.*]] = icmp ult ptr [[LEN1]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]]
+; CHECK-NEXT: [[CONFLICT_RDX13:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT12]]
+; CHECK-NEXT: [[BOUND014:%.*]] = icmp ult ptr [[A]], [[SCEVGEP6]]
+; CHECK-NEXT: [[BOUND115:%.*]] = icmp ult ptr [[LEN2]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT16:%.*]] = and i1 [[BOUND014]], [[BOUND115]]
+; CHECK-NEXT: [[CONFLICT_RDX17:%.*]] = or i1 [[CONFLICT_RDX13]], [[FOUND_CONFLICT16]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX17]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX18]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX18]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[TBAA0]], !alias.scope [[META4:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD19]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP12]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX18]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[TMP15]], [[TMP14]]
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i32 [[ADD3]], ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[LEN1]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[LEN2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP17]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = sext i32 [[ADD]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP18]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP19:![0-9]+]]
+;
+entry:
+ %0 = load i32, ptr %Len1, align 4, !tbaa !5
+ %1 = load i32, ptr %Len2, align 4, !tbaa !5
+ %add10 = add nsw i32 %1, %0
+ %cmp11 = icmp sgt i32 %add10, 0
+ br i1 %cmp11, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %3 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %add3 = add nsw i32 %3, %2
+ %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add3, ptr %arrayidx5, align 4, !tbaa !5
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %4 = load i32, ptr %Len1, align 4, !tbaa !5
+ %5 = load i32, ptr %Len2, align 4, !tbaa !5
+ %add = add nsw i32 %5, %4
+ %6 = sext i32 %add to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %6
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 8, !"PIC Level", i32 2}
+!2 = !{i32 7, !"PIE Level", i32 2}
+!3 = !{i32 7, !"uwtable", i32 2}
+!4 = !{!"clang version 21.1.8"}
+!5 = !{!6, !6, i64 0, i64 4}
+!6 = !{!7, i64 4, !"int"}
+!7 = !{!8, i64 1, !"omnipotent char"}
+!8 = !{!"Simple C++ TBAA"}
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.mustprogress"}
+!11 = !{!"llvm.loop.unroll.disable"}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
+; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
+; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
+; CHECK: [[META11]] = !{[[META5]], [[META8]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META6]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
+; CHECK: [[META15]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META16]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: [[META17]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META18]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META15]], [[META16]], [[META17]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
new file mode 100644
index 0000000000000..1c0495faa4ceb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+;
+; Negative test: the bound is loaded via a volatile (non-simple) load. The detector
+; rejects it (isSafeToHoistBoundLoad requires a simple load), so the loop will be rejected.
+;
+; void foo(int *A, int *B, int *C, volatile int *Len) {
+; for (int i = 0; i < *Len; i++)
+; A[i] = B[i] + C[i];
+; }
+
+define dso_local void @foo(ptr noundef writeonly %A, ptr noundef readonly %B, ptr noundef readonly %C, ptr noundef %Len) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+; CHECK-NOT: .bound.pre
+entry:
+ %len0 = load volatile i32, ptr %Len, align 4
+ %cmp0 = icmp sgt i32 %len0, 0
+ br i1 %cmp0, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx2, align 4
+ %add = add nsw i32 %1, %0
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx4, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %len = load volatile i32, ptr %Len, align 4
+ %ext = sext i32 %len to i64
+ %cmp = icmp slt i64 %indvars.iv.next, %ext
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.unroll.disable"}
>From c6eedcf8f84e44fccdb6f2192016fc63a1e69495 Mon Sep 17 00:00:00 2001
From: Manish Srivastava <manish.srivastava at amd.com>
Date: Fri, 17 Jul 2026 11:30:51 +0530
Subject: [PATCH 2/5] Fix for the prechecks failures
---
.../llvm/Analysis/LoopAccessAnalysis.h | 36 ++++++-----
llvm/include/llvm/Analysis/ScalarEvolution.h | 37 ++++++-----
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 47 ++++++++------
llvm/lib/Analysis/ScalarEvolution.cpp | 64 +++++++++++--------
.../Vectorize/LoopVectorizationLegality.cpp | 3 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +--
6 files changed, 111 insertions(+), 84 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index d84fd0e03a5e9..fd5bf6eea9cd1 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -697,7 +697,8 @@ class LoopAccessInfo {
const TargetLibraryInfo *TLI, AAResults *AA,
DominatorTree *DT, LoopInfo *LI, AssumptionCache *AC,
bool AllowPartial = false,
- ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds = {});
+ ArrayRef<const SCEVTripCountInvariantPredicate *>
+ TripCountInvariantPreds = {});
/// Return true we can analyze the memory accesses in the loop and there are
/// no memory dependence cycles. Note that for dependences between loads &
@@ -717,7 +718,8 @@ class LoopAccessInfo {
/// could be analyzed.
bool hasAllowPartial() const { return AllowPartial; }
- ArrayRef<const SCEVTripCountInvariantPredicate *> getTripCountInvariantPreds() const {
+ ArrayRef<const SCEVTripCountInvariantPredicate *>
+ getTripCountInvariantPreds() const {
return TripCountInvariantPreds;
}
@@ -845,9 +847,10 @@ class LoopAccessInfo {
/// memory accesses could be analyzed.
bool AllowPartial;
- /// Predicates for the loops inside the loop which are used to compute the tripcount and
- /// which this analysis assumes to be invariant.
- SmallVector<const SCEVTripCountInvariantPredicate *, 2> TripCountInvariantPreds;
+ /// Predicates for the loops inside the loop which are used to compute the
+ /// tripcount and which this analysis assumes to be invariant.
+ SmallVector<const SCEVTripCountInvariantPredicate *, 2>
+ TripCountInvariantPreds;
unsigned NumLoads = 0;
unsigned NumStores = 0;
@@ -990,14 +993,15 @@ LLVM_ABI std::pair<const SCEV *, const SCEV *> getStartAndEndForAccess(
DominatorTree *DT, AssumptionCache *AC,
std::optional<ScalarEvolution::LoopGuards> &LoopGuards);
-/// This function will detect whether the loop is uncountabel because the tripcount
-/// is based on a load.
-/// If yes, then it will build a dependency chain of instructions, and a list of loads
-/// which are used to compute the tripcount.
-LLVM_ABI bool collectInvariantLoadsBoundChain(
- Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
- SmallVectorImpl<Instruction *> &HoistedDeps,
- SmallVectorImpl<LoadInst *> &BoundLoads);
+/// This function will detect whether the loop is uncountabel because the
+/// tripcount is based on a load. If yes, then it will build a dependency chain
+/// of instructions, and a list of loads which are used to compute the
+/// tripcount.
+LLVM_ABI bool
+collectInvariantLoadsBoundChain(Loop *L, ScalarEvolution *SE, DominatorTree *DT,
+ AssumptionCache *AC,
+ SmallVectorImpl<Instruction *> &HoistedDeps,
+ SmallVectorImpl<LoadInst *> &BoundLoads);
class LoopAccessInfoManager {
/// The cache.
@@ -1020,8 +1024,10 @@ class LoopAccessInfoManager {
LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial = false);
- LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial,
- ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds);
+ LLVM_ABI const LoopAccessInfo &
+ getInfo(Loop &L, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *>
+ TripCountInvariantPreds);
LLVM_ABI void clear();
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index f39553d3013d2..15ea107d04625 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -464,18 +464,19 @@ class LLVM_ABI SCEVComparePredicate final : public SCEVPredicate {
}
};
-/// This class extends the SCEVPredicate class to represent the assumption that the trip count of a loop is invariant.
-/// This predicate doesn't lower any numeric run-time check but solely depends on the memory alias checks generated
-/// for the loop by LoopVectorizer.
-/// NOTE: Users of this predicate make sure to verify that the memory alias checks are generated for the loop.
+/// This class extends the SCEVPredicate class to represent the assumption that
+/// the trip count of a loop is invariant. This predicate doesn't lower any
+/// numeric run-time check but solely depends on the memory alias checks
+/// generated for the loop by LoopVectorizer. NOTE: Users of this predicate make
+/// sure to verify that the memory alias checks are generated for the loop.
class LLVM_ABI SCEVTripCountInvariantPredicate final : public SCEVPredicate {
const SCEV *TripCountLoad;
const SCEV *TripCountInvariantLoad;
public:
SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
- const SCEV *TripCountLoad,
- const SCEV *TripCountInvariantLoad);
+ const SCEV *TripCountLoad,
+ const SCEV *TripCountInvariantLoad);
/// Implementation of the SCEVPredicate interface
bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override;
@@ -483,12 +484,14 @@ class LLVM_ABI SCEVTripCountInvariantPredicate final : public SCEVPredicate {
bool isAlwaysTrue() const override;
/// This predicate doesn't generate any runtime checks, and solely depends
- /// upon the memory alias checks for the runtime check generation, so contributes
- /// nothing to the check complexity
+ /// upon the memory alias checks for the runtime check generation, so
+ /// contributes nothing to the check complexity
unsigned getComplexity() const override { return 0; }
const SCEV *getTripCountLoad() const { return TripCountLoad; }
- const SCEV *getTripCountInvariantLoad() const { return TripCountInvariantLoad; }
+ const SCEV *getTripCountInvariantLoad() const {
+ return TripCountInvariantLoad;
+ }
static bool classof(const SCEVPredicate *P) {
return P->getKind() == P_TripCountInvariant;
@@ -1153,7 +1156,8 @@ class ScalarEvolution {
LLVM_ABI const SCEV *getPredicatedSymbolicMaxBackedgeTakenCount(
const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates);
- /// To compute the backedge-taken count under the assumption of TripCountInvariant predicate.
+ /// To compute the backedge-taken count under the assumption of
+ /// TripCountInvariant predicate.
LLVM_ABI const SCEV *computeBackedgeTakenCountWithTripCountInvariants(
const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
bool SymbolicMax = false);
@@ -1558,8 +1562,8 @@ class ScalarEvolution {
getWrapPredicate(const SCEVAddRecExpr *AR,
SCEVWrapPredicate::IncrementWrapFlags AddedFlags);
- LLVM_ABI const SCEVPredicate *getTripCountInvariantPredicate(const SCEV *Load,
- const SCEV *InvariantLoad);
+ LLVM_ABI const SCEVPredicate *
+ getTripCountInvariantPredicate(const SCEV *Load, const SCEV *InvariantLoad);
/// Re-writes the SCEV according to the Predicates in \p A.
LLVM_ABI const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L,
@@ -1929,7 +1933,8 @@ class ScalarEvolution {
/// function as they are computed.
DenseMap<const Loop *, BackedgeTakenInfo> PredicatedBackedgeTakenCounts;
- DenseMap<const Value *, const SCEV *> *ActiveTripCountInvariantSubst = nullptr;
+ DenseMap<const Value *, const SCEV *> *ActiveTripCountInvariantSubst =
+ nullptr;
/// Loops whose backedge taken counts directly use this non-constant SCEV.
DenseMap<const SCEV *, SmallPtrSet<PointerIntPair<const Loop *, 1, bool>, 4>>
@@ -2703,7 +2708,8 @@ class PredicatedScalarEvolution {
return !TripCountInvariantPreds.empty();
}
- LLVM_ABI ArrayRef<const SCEVTripCountInvariantPredicate *> getTripCountInvariantPredicates() const {
+ LLVM_ABI ArrayRef<const SCEVTripCountInvariantPredicate *>
+ getTripCountInvariantPredicates() const {
return TripCountInvariantPreds;
}
@@ -2778,7 +2784,8 @@ class PredicatedScalarEvolution {
/// The symbolic backedge taken count.
const SCEV *SymbolicMaxBackedgeCount = nullptr;
- SmallVector<const SCEVTripCountInvariantPredicate *, 2> TripCountInvariantPreds;
+ SmallVector<const SCEVTripCountInvariantPredicate *, 2>
+ TripCountInvariantPreds;
DenseMap<const Value *, const SCEV *> TripCountInvariantMap;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index aecb316fd162a..fe2ea5201b2d9 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -25,9 +25,9 @@
#include "llvm/Analysis/AliasSetTracker.h"
#include "llvm/Analysis/AssumeBundleQueries.h"
#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
-#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -3204,15 +3204,17 @@ static bool isSafeToHoistBoundLoad(Instruction *I,
if (ModifiedPtrs.count(Ptr)) {
return false;
}
- if (!isBoundLoadSafeToLoadAtCtx(Ptr, LI->getType(), LI->getAlign(), CtxI, DT,
- AC)) {
+ if (!isBoundLoadSafeToLoadAtCtx(Ptr, LI->getType(), LI->getAlign(), CtxI,
+ DT, AC)) {
return false;
}
return true;
}
- // Limiting down the kind of instructions we can hoist to avoid any unforeseen side effects.
- if (I->isBinaryOp() || I->isUnaryOp() || I->isCast() || isa<GetElementPtrInst>(I)) {
+ // Limiting down the kind of instructions we can hoist to avoid any unforeseen
+ // side effects.
+ if (I->isBinaryOp() || I->isUnaryOp() || I->isCast() ||
+ isa<GetElementPtrInst>(I)) {
return isSafeToSpeculativelyExecute(I, CtxI, AC, DT);
}
return false;
@@ -3255,7 +3257,8 @@ LLVM_ABI bool llvm::collectInvariantLoadsBoundChain(
/// Anything written inside the loop is a "not-invariant" pointer; the bound
/// load cannot share a pointer with such writes.
- /// This is to prune out cases like for (i = 0; i < *Len; ++i) { Len[i] = ..*.. }
+ /// This is to prune out cases like for (i = 0; i < *Len; ++i) { Len[i] =
+ /// ..*.. }
SmallPtrSet<Value *, 16> ModifiedPtrs;
for (BasicBlock *BB : L->getBlocks()) {
for (Instruction &I : *BB) {
@@ -3324,18 +3327,18 @@ LLVM_ABI bool llvm::collectInvariantLoadsBoundChain(
return !BoundLoads.empty() && !HoistedDeps.empty();
}
-LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
- const TargetTransformInfo *TTI,
- const TargetLibraryInfo *TLI, AAResults *AA,
- DominatorTree *DT, LoopInfo *LI,
- AssumptionCache *AC, bool AllowPartial,
- ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds)
+LoopAccessInfo::LoopAccessInfo(
+ Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI,
+ const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT,
+ LoopInfo *LI, AssumptionCache *AC, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds)
: PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial),
TripCountInvariantPreds(TripCountInvariantPreds) {
for (const SCEVTripCountInvariantPredicate *P : TripCountInvariantPreds) {
- PSE->addTripCountInvariantPredicate(P->getTripCountLoad(), P->getTripCountInvariantLoad());
+ PSE->addTripCountInvariantPredicate(P->getTripCountLoad(),
+ P->getTripCountInvariantLoad());
}
unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
@@ -3413,24 +3416,28 @@ const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
return getInfo(L, AllowPartial, {});
}
-const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
- bool AllowPartial,
- ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds) {
- const auto &[It, Inserted] = LoopAccessInfoMap.try_emplace(&L);
+const LoopAccessInfo &LoopAccessInfoManager::getInfo(
+ Loop &L, bool AllowPartial,
+ ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds) {
+ auto Insertion = LoopAccessInfoMap.try_emplace(&L);
+ auto It = Insertion.first;
+ bool Inserted = Insertion.second;
auto SamePreds = [&]() {
ArrayRef<const SCEVTripCountInvariantPredicate *> Cached =
It->second->getTripCountInvariantPreds();
if (Cached.size() != TripCountInvariantPreds.size())
return false;
- return std::equal(Cached.begin(), Cached.end(), TripCountInvariantPreds.begin());
+ return std::equal(Cached.begin(), Cached.end(),
+ TripCountInvariantPreds.begin());
};
// We need to create the LoopAccessInfo if either we don't already have one,
// or if it was created with a different value of AllowPartial.
if (Inserted || It->second->hasAllowPartial() != AllowPartial || !SamePreds())
- It->second = std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT,
- &LI, AC, AllowPartial, TripCountInvariantPreds);
+ It->second =
+ std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT, &LI, AC,
+ AllowPartial, TripCountInvariantPreds);
return *It->second;
}
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index a204c46947001..02926865366cf 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8695,7 +8695,8 @@ const SCEV *ScalarEvolution::computeBackedgeTakenCountWithTripCountInvariants(
SaveAndRestore<DenseMap<const Value *, const SCEV *> *> Active(
ActiveTripCountInvariantSubst, &Subst);
- BackedgeTakenInfo BTI = computeBackedgeTakenCount(L, /*AllowPredicates=*/false);
+ BackedgeTakenInfo BTI =
+ computeBackedgeTakenCount(L, /*AllowPredicates=*/false);
return SymbolicMax ? BTI.getSymbolicMax(L, this) : BTI.getExact(L, this);
}
@@ -9487,10 +9488,12 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
- // If the TripCountInvariant predicate is active, then replace the LHS and RHS to their proper
- // substition values as per the substitution map.
- if (ActiveTripCountInvariantSubst && !ActiveTripCountInvariantSubst->empty()) {
- auto &Subst = const_cast<ValueToSCEVMapTy &>(*ActiveTripCountInvariantSubst);
+ // If the TripCountInvariant predicate is active, then replace the LHS and RHS
+ // to their proper substition values as per the substitution map.
+ if (ActiveTripCountInvariantSubst &&
+ !ActiveTripCountInvariantSubst->empty()) {
+ auto &Subst =
+ const_cast<ValueToSCEVMapTy &>(*ActiveTripCountInvariantSubst);
LHS = SCEVParameterRewriter::rewrite(LHS, *this, Subst);
RHS = SCEVParameterRewriter::rewrite(RHS, *this, Subst);
@@ -15295,8 +15298,9 @@ const SCEVPredicate *ScalarEvolution::getWrapPredicate(
return OF;
}
-const SCEVPredicate *ScalarEvolution::getTripCountInvariantPredicate(const SCEV *Load,
- const SCEV *InvariantLoad) {
+const SCEVPredicate *
+ScalarEvolution::getTripCountInvariantPredicate(const SCEV *Load,
+ const SCEV *InvariantLoad) {
FoldingSetNodeID ID;
ID.AddInteger(SCEVPredicate::P_TripCountInvariant);
ID.AddPointer(Load);
@@ -15304,8 +15308,8 @@ const SCEVPredicate *ScalarEvolution::getTripCountInvariantPredicate(const SCEV
void *IP = nullptr;
if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
return S;
- auto *TCI = new (SCEVAllocator)
- SCEVTripCountInvariantPredicate(ID.Intern(SCEVAllocator), Load, InvariantLoad);
+ auto *TCI = new (SCEVAllocator) SCEVTripCountInvariantPredicate(
+ ID.Intern(SCEVAllocator), Load, InvariantLoad);
UniquePreds.InsertNode(TCI, IP);
return TCI;
}
@@ -15517,27 +15521,30 @@ void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
}
-SCEVTripCountInvariantPredicate::SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
- const SCEV *TripCountLoad,
- const SCEV *TripCountInvariantLoad)
- : SCEVPredicate(ID, P_TripCountInvariant), TripCountLoad(TripCountLoad), TripCountInvariantLoad(TripCountInvariantLoad) {}
+SCEVTripCountInvariantPredicate::SCEVTripCountInvariantPredicate(
+ const FoldingSetNodeIDRef ID, const SCEV *TripCountLoad,
+ const SCEV *TripCountInvariantLoad)
+ : SCEVPredicate(ID, P_TripCountInvariant), TripCountLoad(TripCountLoad),
+ TripCountInvariantLoad(TripCountInvariantLoad) {}
bool SCEVTripCountInvariantPredicate::implies(const SCEVPredicate *N,
ScalarEvolution &SE) const {
const auto *Op = dyn_cast<SCEVTripCountInvariantPredicate>(N);
if (!Op)
return false;
- return Op->TripCountLoad == TripCountLoad && Op->TripCountInvariantLoad == TripCountInvariantLoad;
+ return Op->TripCountLoad == TripCountLoad &&
+ Op->TripCountInvariantLoad == TripCountInvariantLoad;
}
bool SCEVTripCountInvariantPredicate::isAlwaysTrue() const {
return TripCountLoad == TripCountInvariantLoad;
}
-void SCEVTripCountInvariantPredicate::print(raw_ostream &OS, unsigned Depth) const {
- OS.indent(Depth) << "Trip count invariant predicate: " << *TripCountLoad
- << " is invariant (== " << *TripCountInvariantLoad << ")\n"
- << "\n";
+void SCEVTripCountInvariantPredicate::print(raw_ostream &OS,
+ unsigned Depth) const {
+ OS.indent(Depth) << "Trip count invariant predicate: " << *TripCountLoad
+ << " is invariant (== " << *TripCountInvariantLoad << ")\n"
+ << "\n";
}
SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
@@ -15756,7 +15763,8 @@ const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds);
if (!TripCountInvariantMap.empty()) {
- NewSCEV = SCEVParameterRewriter::rewrite(NewSCEV, SE, TripCountInvariantMap);
+ NewSCEV =
+ SCEVParameterRewriter::rewrite(NewSCEV, SE, TripCountInvariantMap);
}
Entry = {Generation, NewSCEV};
@@ -15767,10 +15775,9 @@ const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
if (!BackedgeCount) {
if (!TripCountInvariantPreds.empty()) {
- BackedgeCount = SE.computeBackedgeTakenCountWithTripCountInvariants(&L,
- TripCountInvariantPreds, /*SymbolicMax=*/false);
- }
- else {
+ BackedgeCount = SE.computeBackedgeTakenCountWithTripCountInvariants(
+ &L, TripCountInvariantPreds, /*SymbolicMax=*/false);
+ } else {
SmallVector<const SCEVPredicate *, 4> Preds;
BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
for (const auto *P : Preds)
@@ -15783,10 +15790,10 @@ const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
if (!SymbolicMaxBackedgeCount) {
if (!TripCountInvariantPreds.empty()) {
- SymbolicMaxBackedgeCount = SE.computeBackedgeTakenCountWithTripCountInvariants(&L,
- TripCountInvariantPreds, /*SymbolicMax=*/true);
- }
- else {
+ SymbolicMaxBackedgeCount =
+ SE.computeBackedgeTakenCountWithTripCountInvariants(
+ &L, TripCountInvariantPreds, /*SymbolicMax=*/true);
+ } else {
SmallVector<const SCEVPredicate *, 4> Preds;
SymbolicMaxBackedgeCount =
SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
@@ -15817,7 +15824,8 @@ void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
updateGeneration();
}
-void PredicatedScalarEvolution::addTripCountInvariantPredicate(const SCEV *Load, const SCEV *InvariantLoad) {
+void PredicatedScalarEvolution::addTripCountInvariantPredicate(
+ const SCEV *Load, const SCEV *InvariantLoad) {
const auto *V = dyn_cast<SCEVUnknown>(Load);
assert(V && "mem-invariant Load must be a SCEVUnknown");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 94e101297f68a..ddae25fc17386 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1158,8 +1158,7 @@ bool LoopVectorizationLegality::canVectorizeIndirectUnsafeDependences() {
}
bool LoopVectorizationLegality::canVectorizeMemory() {
- LAI = &LAIs.getInfo(*TheLoop, false,
- PSE.getTripCountInvariantPredicates());
+ LAI = &LAIs.getInfo(*TheLoop, false, PSE.getTripCountInvariantPredicates());
const OptimizationRemarkAnalysis *LAR = LAI->getReport();
if (LAR) {
ORE->emit([&]() {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7dbd53a83904f..95a0cd21a7291 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1643,8 +1643,7 @@ class GeneratedRTChecks {
SCEVPredsVec.push_back(P);
}
}
- }
- else if (!isa<SCEVTripCountInvariantPredicate>(&UnionPred)) {
+ } else if (!isa<SCEVTripCountInvariantPredicate>(&UnionPred)) {
SCEVPredsVec.push_back(&UnionPred);
}
SCEVUnionPredicate FilteredPred(SCEVPredsVec, *PSE.getSE());
@@ -7882,7 +7881,7 @@ bool EnableLoadBoundVectorization(Loop *L, PredicatedScalarEvolution &PSE,
ScalarEvolution *SE, DominatorTree *DT,
AssumptionCache *AC) {
if (!L->isInnermost() || !L->isLoopSimplifyForm() ||
- L->getNumBackEdges() != 1 || !L->getUniqueExitBlock()) {
+ L->getNumBackEdges() != 1 || !L->getUniqueExitBlock()) {
return false;
}
@@ -7892,7 +7891,8 @@ bool EnableLoadBoundVectorization(Loop *L, PredicatedScalarEvolution &PSE,
SmallVector<Instruction *, 16> HoistedDeps;
SmallVector<LoadInst *, 4> BoundLoads;
- if (!collectInvariantLoadsBoundChain(L, SE, DT, AC, HoistedDeps, BoundLoads)) {
+ if (!collectInvariantLoadsBoundChain(L, SE, DT, AC, HoistedDeps,
+ BoundLoads)) {
return false;
}
>From 58bdb35ef434d164427b7463f63ec3f6ae8341bd Mon Sep 17 00:00:00 2001
From: Manish Srivastava <manish.srivastava at amd.com>
Date: Fri, 17 Jul 2026 12:35:08 +0530
Subject: [PATCH 3/5] Changes related to making the LIT test minimal
---
.../dynamic-bound-array-element.ll | 31 +---
.../dynamic-bound-chained-loads.ll | 37 ++---
.../LoopVectorize/dynamic-bound-fcmp-exit.ll | 10 +-
.../LoopVectorize/dynamic-bound-lambda.ll | 124 ++++++---------
.../dynamic-bound-loop-varying.ll | 29 +---
.../LoopVectorize/dynamic-bound-simple.ll | 125 ++++++---------
.../dynamic-bound-stored-pointer.ll | 10 +-
.../LoopVectorize/dynamic-bound-two-loads.ll | 147 ++++++++----------
.../dynamic-bound-volatile-load.ll | 10 +-
9 files changed, 194 insertions(+), 329 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 7d8db5c67f519..9de99bfede967 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -12,12 +12,12 @@
; }
;
-define dso_local void @foo(ptr noundef captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %Len) #0 {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
entry:
- %arrayidx = getelementptr inbounds nuw i8, ptr %A, i64 8
- %0 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %arrayidx = getelementptr inbounds nuw i32, ptr %A, i64 2
+ %0 = load i32, ptr %arrayidx, align 4
%cmp11 = icmp sgt i32 %0, 0
br i1 %cmp11, label %for.body, label %for.cond.cleanup
@@ -27,30 +27,15 @@ for.cond.cleanup:
for.body:
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx1 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
- %1 = load i32, ptr %arrayidx1, align 4, !tbaa !5
+ %1 = load i32, ptr %arrayidx1, align 4
%arrayidx3 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
- %2 = load i32, ptr %arrayidx3, align 4, !tbaa !5
+ %2 = load i32, ptr %arrayidx3, align 4
%add = add nsw i32 %2, %1
%arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
- store i32 %add, ptr %arrayidx5, align 4, !tbaa !5
+ store i32 %add, ptr %arrayidx5, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %3 = load i32, ptr %arrayidx, align 4
%4 = sext i32 %3 to i64
%cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 4}
-!6 = !{!7, i64 4, !"int"}
-!7 = !{!8, i64 1, !"omnipotent char"}
-!8 = !{!"Simple C++ TBAA"}
-!9 = distinct !{!9, !10, !11}
-!10 = !{!"llvm.loop.mustprogress"}
-!11 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
index b9a7ad48547bf..e53cf9b9ee738 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
@@ -5,19 +5,18 @@
; load (load i32, ptr %ptr) because %ptr itself is loaded inside the loop.
; The loop will be rejected.
;
-; C source:
; void foo(int *A, int *B, int *C, int **PtrLen) {
; for (int i = 0; i < **PtrLen; i++)
; A[i] = B[i] + C[i];
; }
-define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %PtrLen) #0 {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %PtrLen) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- %ptr0 = load ptr, ptr %PtrLen, align 8, !tbaa !5
- %val0 = load i32, ptr %ptr0, align 4, !tbaa !10
+ %ptr0 = load ptr, ptr %PtrLen, align 8
+ %val0 = load i32, ptr %ptr0, align 4
%cmp9 = icmp sgt i32 %val0, 0
br i1 %cmp9, label %for.body, label %for.cond.cleanup
@@ -27,34 +26,16 @@ for.cond.cleanup:
for.body:
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
- %1 = load i32, ptr %arrayidx, align 4, !tbaa !10
+ %1 = load i32, ptr %arrayidx, align 4
%arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
- %2 = load i32, ptr %arrayidx2, align 4, !tbaa !10
+ %2 = load i32, ptr %arrayidx2, align 4
%add = add nsw i32 %2, %1
%arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
- store i32 %add, ptr %arrayidx4, align 4, !tbaa !10
+ store i32 %add, ptr %arrayidx4, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %ptr = load ptr, ptr %PtrLen, align 8, !tbaa !5
- %val = load i32, ptr %ptr, align 4, !tbaa !10
+ %ptr = load ptr, ptr %PtrLen, align 8
+ %val = load i32, ptr %ptr, align 4
%3 = sext i32 %val to i64
%cmp = icmp slt i64 %indvars.iv.next, %3
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !12
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 8}
-!6 = !{!7, i64 8, !"p1 int"}
-!7 = !{!8, i64 8, !"any pointer"}
-!8 = !{!9, i64 1, !"omnipotent char"}
-!9 = !{!"Simple C++ TBAA"}
-!10 = !{!11, !11, i64 0, i64 4}
-!11 = !{!8, i64 4, !"int"}
-!12 = distinct !{!12, !13, !14}
-!13 = !{!"llvm.loop.mustprogress"}
-!14 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
index d10ea005bf350..b039314b24297 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
@@ -10,7 +10,7 @@
; A[(int)x] = B[(int)x];
; }
-define dso_local void @foo(ptr noundef %A, ptr noundef readonly %B, ptr noundef readonly %Limit) #0 {
+define void @foo(ptr %A, ptr %B, ptr %Limit) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
@@ -28,14 +28,8 @@ for.body:
%fiv.next = fadd float %fiv, 1.0
%lim = load float, ptr %Limit, align 4
%cmp = fcmp olt float %fiv.next, %lim
- br i1 %cmp, label %for.body, label %for.exit, !llvm.loop !0
+ br i1 %cmp, label %for.body, label %for.exit
for.exit:
ret void
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.mustprogress"}
-!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
index e43a7bcb15738..6f86515e1a867 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
;
; Lambda with loop bound loaded from pointer (*len).
@@ -12,26 +12,25 @@
; }
;
-define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readnone captures(none) %D, ptr noundef readonly captures(none) %len) #0 {
-;
-; CHECK-LABEL: define dso_local void @foo(
-; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readnone captures(none) [[D:%.*]], ptr noundef readonly captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %D, ptr %len) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[LEN:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[CMP10_I:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP10_I]], label %[[FOR_BODY_I_PREHEADER:.*]], label %[[EXIT:.*]]
; CHECK: [[FOR_BODY_I_PREHEADER]]:
-; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
-; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[LEN]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
@@ -46,23 +45,23 @@ define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef
; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT10]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[TBAA0]], !alias.scope [[META4:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
-; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD13]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_I_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
@@ -70,78 +69,57 @@ define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef
; CHECK: [[FOR_BODY_I]]:
; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_I:%.*]], %[[FOR_BODY_I]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_I]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX_I]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX_I]], align 4
; CHECK-NEXT: [[ARRAYIDX3_I:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV_I]]
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX3_I]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[MUL_I:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX3_I]], align 4
+; CHECK-NEXT: [[MUL_I:%.*]] = mul nsw i32 [[TMP9]], [[TMP8]]
; CHECK-NEXT: [[ARRAYIDX5_I:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV_I]]
-; CHECK-NEXT: store i32 [[MUL_I]], ptr [[ARRAYIDX5_I]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store i32 [[MUL_I]], ptr [[ARRAYIDX5_I]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nuw nsw i64 [[INDVARS_IV_I]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
-; CHECK-NEXT: [[CMP_I:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_I]], [[TMP10]]
-; CHECK-NEXT: br i1 [[CMP_I]], label %[[FOR_BODY_I]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[LEN]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT: [[CMP_I:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_I]], [[TMP11]]
+; CHECK-NEXT: br i1 [[CMP_I]], label %[[FOR_BODY_I]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
- %0 = load i32, ptr %len, align 4, !tbaa !5
+ %0 = load i32, ptr %len, align 4
%cmp10.i = icmp sgt i32 %0, 0
br i1 %cmp10.i, label %for.body.i, label %exit
for.body.i:
%indvars.iv.i = phi i64 [ %indvars.iv.next.i, %for.body.i ], [ 0, %entry ]
%arrayidx.i = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv.i
- %1 = load i32, ptr %arrayidx.i, align 4, !tbaa !5
+ %1 = load i32, ptr %arrayidx.i, align 4
%arrayidx3.i = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv.i
- %2 = load i32, ptr %arrayidx3.i, align 4, !tbaa !5
+ %2 = load i32, ptr %arrayidx3.i, align 4
%mul.i = mul nsw i32 %2, %1
%arrayidx5.i = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv.i
- store i32 %mul.i, ptr %arrayidx5.i, align 4, !tbaa !5
+ store i32 %mul.i, ptr %arrayidx5.i, align 4
%indvars.iv.next.i = add nuw nsw i64 %indvars.iv.i, 1
- %3 = load i32, ptr %len, align 4, !tbaa !5
+ %3 = load i32, ptr %len, align 4
%4 = sext i32 %3 to i64
%cmp.i = icmp slt i64 %indvars.iv.next.i, %4
- br i1 %cmp.i, label %for.body.i, label %exit, !llvm.loop !9
+ br i1 %cmp.i, label %for.body.i, label %exit
exit:
ret void
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 4}
-!6 = !{!7, i64 4, !"int"}
-!7 = !{!8, i64 1, !"omnipotent char"}
-!8 = !{!"Simple C++ TBAA"}
-!9 = distinct !{!9, !10, !11}
-!10 = !{!"llvm.loop.mustprogress"}
-!11 = !{!"llvm.loop.unroll.disable"}
;.
-; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
-; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
-; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
-; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
-; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
-; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
-; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
-; CHECK: [[META11]] = !{[[META5]], [[META8]], [[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
-; CHECK: [[META14]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META15]] = !{!"llvm.loop.unroll.disable"}
-; CHECK: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META14]], [[META15]], [[META16]]}
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
+; CHECK: [[META7]] = !{[[META1]], [[META4]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META11]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META10]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
index 5ad94b333f144..0942f94be1fde 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
@@ -9,12 +9,12 @@
; A[i] = B[i] + C[i];
; }
-define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len) #0 {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- %0 = load i32, ptr %Len, align 4, !tbaa !5
+ %0 = load i32, ptr %Len, align 4
%cmp12 = icmp sgt i32 %0, 0
br i1 %cmp12, label %for.body, label %for.cond.cleanup
@@ -24,31 +24,16 @@ for.cond.cleanup:
for.body:
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx2 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
- %1 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %1 = load i32, ptr %arrayidx2, align 4
%arrayidx4 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
- %2 = load i32, ptr %arrayidx4, align 4, !tbaa !5
+ %2 = load i32, ptr %arrayidx4, align 4
%add = add nsw i32 %2, %1
%arrayidx6 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
- store i32 %add, ptr %arrayidx6, align 4, !tbaa !5
+ store i32 %add, ptr %arrayidx6, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
%arrayidx = getelementptr inbounds nuw i32, ptr %Len, i64 %indvars.iv.next
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %3 = load i32, ptr %arrayidx, align 4
%4 = sext i32 %3 to i64
%cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 4}
-!6 = !{!7, i64 4, !"int"}
-!7 = !{!8, i64 1, !"omnipotent char"}
-!8 = !{!"Simple C++ TBAA"}
-!9 = distinct !{!9, !10, !11}
-!10 = !{!"llvm.loop.mustprogress"}
-!11 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index c6e6d344e2fec..58a6e65dd5f47 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
;
; Simple loop with loop bound loaded from a pointer (*Len).
@@ -7,28 +7,26 @@
; for (int i = 0; i < *Len; i++)
; A[i] = B[i] + C[i];
; }
-;
-define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len) #0 {
-;
-; CHECK-LABEL: define dso_local void @foo(
-; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readonly captures(none) [[LEN:%.*]]) #[[ATTR0:[0-9]+]] {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[LEN:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
-; CHECK-NEXT: [[SMAX12:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX12]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP2]]
; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[LEN]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]
@@ -43,23 +41,23 @@ define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef
; CHECK-NEXT: [[CONFLICT_RDX11:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT10]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX12]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX12]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !tbaa [[TBAA0]], !alias.scope [[META4:![0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
-; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD13]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD12]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX12]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
@@ -71,20 +69,20 @@ define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP9]], [[TMP8]]
; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[LEN]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP10]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[LEN]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP11]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]]
;
entry:
- %0 = load i32, ptr %Len, align 4, !tbaa !5
+ %0 = load i32, ptr %Len, align 4
%cmp9 = icmp sgt i32 %0, 0
br i1 %cmp9, label %for.body, label %for.cond.cleanup
@@ -94,51 +92,30 @@ for.cond.cleanup:
for.body:
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
- %1 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %1 = load i32, ptr %arrayidx, align 4
%arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
- %2 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %2 = load i32, ptr %arrayidx2, align 4
%add = add nsw i32 %2, %1
%arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
- store i32 %add, ptr %arrayidx4, align 4, !tbaa !5
+ store i32 %add, ptr %arrayidx4, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %3 = load i32, ptr %Len, align 4, !tbaa !5
+ %3 = load i32, ptr %Len, align 4
%4 = sext i32 %3 to i64
%cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 4}
-!6 = !{!7, i64 4, !"int"}
-!7 = !{!8, i64 1, !"omnipotent char"}
-!8 = !{!"Simple C++ TBAA"}
-!9 = distinct !{!9, !10, !11}
-!10 = !{!"llvm.loop.mustprogress"}
-!11 = !{!"llvm.loop.unroll.disable"}
;.
-; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
-; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
-; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
-; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
-; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
-; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
-; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
-; CHECK: [[META11]] = !{[[META5]], [[META8]], [[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
-; CHECK: [[META14]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META15]] = !{!"llvm.loop.unroll.disable"}
-; CHECK: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META14]], [[META15]], [[META16]]}
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
+; CHECK: [[META7]] = !{[[META1]], [[META4]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META11]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META10]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
index ecb087261e3af..a1a0ed6ba277b 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
@@ -11,7 +11,7 @@
; }
; }
-define dso_local void @foo(ptr noundef %A, ptr noundef readonly %B, ptr noundef readonly %C, ptr noundef %Len) #0 {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
@@ -37,11 +37,5 @@ for.body:
%len = load i32, ptr %Len, align 4
%ext = sext i32 %len to i64
%cmp = icmp slt i64 %indvars.iv.next, %ext
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.mustprogress"}
-!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
index 240f4f0143e95..ddfb275a7dbd9 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -passes=loop-vectorize -enable-vectorize-loads-as-bound -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
;
; Loop bound is *Len1 + *Len2. Two loads and an add are hoisted to the preheader for the dynamic bound.
@@ -7,38 +7,36 @@
; for (int i = 0; i < *Len1 + *Len2; i++)
; A[i] = B[i] + C[i];
; }
-;
-define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef readonly captures(none) %B, ptr noundef readonly captures(none) %C, ptr noundef readonly captures(none) %Len1, ptr noundef readonly captures(none) %Len2) #0 {
-;
-; CHECK-LABEL: define dso_local void @foo(
-; CHECK-SAME: ptr noundef writeonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef readonly captures(none) [[C:%.*]], ptr noundef readonly captures(none) [[LEN1:%.*]], ptr noundef readonly captures(none) [[LEN2:%.*]]) #[[ATTR0:[0-9]+]] {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len1, ptr %Len2) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[LEN1:%.*]], ptr [[LEN2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[LEN2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN1]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[LEN2]], align 4
; CHECK-NEXT: [[ADD10:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[ADD10]], 0
; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
-; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN1]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = load i32, ptr [[LEN2]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN1]], align 4
+; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = load i32, ptr [[LEN2]], align 4
; CHECK-NEXT: [[ADD_BOUND_PRE:%.*]] = add nsw i32 [[DOTBOUND_PRE1]], [[DOTBOUND_PRE]]
; CHECK-NEXT: [[DOTBOUND_PRE2:%.*]] = sext i32 [[ADD_BOUND_PRE]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[DOTBOUND_PRE1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[SMAX18:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX18]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP5]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[DOTBOUND_PRE1]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP7]], i64 1)
-; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[SMAX]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[DOTBOUND_PRE1]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP8]], i64 1)
+; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[SMAX]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP9]]
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[LEN1]], i64 4
; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[LEN2]], i64 4
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
@@ -58,23 +56,23 @@ define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef
; CHECK-NEXT: [[CONFLICT_RDX17:%.*]] = or i1 [[CONFLICT_RDX13]], [[FOUND_CONFLICT16]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX17]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX18]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX18]], [[N_MOD_VF]]
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP5]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP5]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4, !tbaa [[TBAA0]], !alias.scope [[META4:![0-9]+]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
-; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[WIDE_LOAD19]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP12]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i32>, ptr [[TMP11]], align 4, !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[WIDE_LOAD18]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP12]], ptr [[TMP13]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX18]], [[N_VEC]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
@@ -86,23 +84,23 @@ define dso_local void @foo(ptr noundef writeonly captures(none) %A, ptr noundef
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[TMP15]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[TMP16]], [[TMP15]]
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT: store i32 [[ADD3]], ptr [[ARRAYIDX5]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT: store i32 [[ADD3]], ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[LEN1]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[LEN2]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP17]], [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = sext i32 [[ADD]] to i64
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP18]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[LEN1]], align 4
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[LEN2]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP18]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[ADD]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP19]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP13:![0-9]+]]
;
entry:
- %0 = load i32, ptr %Len1, align 4, !tbaa !5
- %1 = load i32, ptr %Len2, align 4, !tbaa !5
+ %0 = load i32, ptr %Len1, align 4
+ %1 = load i32, ptr %Len2, align 4
%add10 = add nsw i32 %1, %0
%cmp11 = icmp sgt i32 %add10, 0
br i1 %cmp11, label %for.body, label %for.cond.cleanup
@@ -113,54 +111,33 @@ for.cond.cleanup:
for.body:
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
%arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
- %2 = load i32, ptr %arrayidx, align 4, !tbaa !5
+ %2 = load i32, ptr %arrayidx, align 4
%arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
- %3 = load i32, ptr %arrayidx2, align 4, !tbaa !5
+ %3 = load i32, ptr %arrayidx2, align 4
%add3 = add nsw i32 %3, %2
%arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
- store i32 %add3, ptr %arrayidx5, align 4, !tbaa !5
+ store i32 %add3, ptr %arrayidx5, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %4 = load i32, ptr %Len1, align 4, !tbaa !5
- %5 = load i32, ptr %Len2, align 4, !tbaa !5
+ %4 = load i32, ptr %Len1, align 4
+ %5 = load i32, ptr %Len2, align 4
%add = add nsw i32 %5, %4
%6 = sext i32 %add to i64
%cmp = icmp slt i64 %indvars.iv.next, %6
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !9
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 8, !"PIC Level", i32 2}
-!2 = !{i32 7, !"PIE Level", i32 2}
-!3 = !{i32 7, !"uwtable", i32 2}
-!4 = !{!"clang version 21.1.8"}
-!5 = !{!6, !6, i64 0, i64 4}
-!6 = !{!7, i64 4, !"int"}
-!7 = !{!8, i64 1, !"omnipotent char"}
-!8 = !{!"Simple C++ TBAA"}
-!9 = distinct !{!9, !10, !11}
-!10 = !{!"llvm.loop.mustprogress"}
-!11 = !{!"llvm.loop.unroll.disable"}
;.
-; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 4}
-; CHECK: [[META1]] = !{[[META2:![0-9]+]], i64 4, !"int"}
-; CHECK: [[META2]] = !{[[META3:![0-9]+]], i64 1, !"omnipotent char"}
-; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
-; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
-; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
-; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
-; CHECK: [[META11]] = !{[[META5]], [[META8]], [[META12:![0-9]+]], [[META13:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
-; CHECK: [[META13]] = distinct !{[[META13]], [[META6]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]], [[META18:![0-9]+]]}
-; CHECK: [[META15]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META16]] = !{!"llvm.loop.unroll.disable"}
-; CHECK: [[META17]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META18]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META15]], [[META16]], [[META17]]}
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
+; CHECK: [[META7]] = !{[[META1]], [[META4]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META2]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]]}
+; CHECK: [[META11]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META12]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META11]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
index 1c0495faa4ceb..0441064259bf6 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
@@ -8,7 +8,7 @@
; A[i] = B[i] + C[i];
; }
-define dso_local void @foo(ptr noundef writeonly %A, ptr noundef readonly %B, ptr noundef readonly %C, ptr noundef %Len) #0 {
+define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
@@ -33,11 +33,5 @@ for.body:
%len = load volatile i32, ptr %Len, align 4
%ext = sext i32 %len to i64
%cmp = icmp slt i64 %indvars.iv.next, %ext
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
}
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable }
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.mustprogress"}
-!2 = !{!"llvm.loop.unroll.disable"}
>From d707b6832f7062d9178d566f63d88134e575216c Mon Sep 17 00:00:00 2001
From: Manish Srivastava <manish.srivastava at amd.com>
Date: Wed, 5 Aug 2026 11:51:59 +0530
Subject: [PATCH 4/5] Removed the TripCountInvariantPredicate
Based on the review comments, the TripCountInvariantPredicate is removed
from the ScalarEvolution and LoopAccessAnalysis. Now same functionality
is achieved by existing SCEV's compare predicate.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 17 +-
llvm/include/llvm/Analysis/ScalarEvolution.h | 75 ++------
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 52 +++---
llvm/lib/Analysis/ScalarEvolution.cpp | 167 +++++-------------
.../Utils/ScalarEvolutionExpander.cpp | 2 -
.../Vectorize/LoopVectorizationLegality.cpp | 2 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 18 +-
7 files changed, 95 insertions(+), 238 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index fd5bf6eea9cd1..b466fda78ff93 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -697,8 +697,7 @@ class LoopAccessInfo {
const TargetLibraryInfo *TLI, AAResults *AA,
DominatorTree *DT, LoopInfo *LI, AssumptionCache *AC,
bool AllowPartial = false,
- ArrayRef<const SCEVTripCountInvariantPredicate *>
- TripCountInvariantPreds = {});
+ ArrayRef<const SCEVPredicate *> Assumptions = {});
/// Return true we can analyze the memory accesses in the loop and there are
/// no memory dependence cycles. Note that for dependences between loads &
@@ -718,10 +717,7 @@ class LoopAccessInfo {
/// could be analyzed.
bool hasAllowPartial() const { return AllowPartial; }
- ArrayRef<const SCEVTripCountInvariantPredicate *>
- getTripCountInvariantPreds() const {
- return TripCountInvariantPreds;
- }
+ ArrayRef<const SCEVPredicate *> getAssumptions() const { return Assumptions; }
const RuntimePointerChecking *getRuntimePointerChecking() const {
return PtrRtChecking.get();
@@ -847,10 +843,8 @@ class LoopAccessInfo {
/// memory accesses could be analyzed.
bool AllowPartial;
- /// Predicates for the loops inside the loop which are used to compute the
- /// tripcount and which this analysis assumes to be invariant.
- SmallVector<const SCEVTripCountInvariantPredicate *, 2>
- TripCountInvariantPreds;
+ /// Assumptions under which analysis is constructed
+ SmallVector<const SCEVPredicate *, 2> Assumptions;
unsigned NumLoads = 0;
unsigned NumStores = 0;
@@ -1026,8 +1020,7 @@ class LoopAccessInfoManager {
LLVM_ABI const LoopAccessInfo &
getInfo(Loop &L, bool AllowPartial,
- ArrayRef<const SCEVTripCountInvariantPredicate *>
- TripCountInvariantPreds);
+ ArrayRef<const SCEVPredicate *> Assumptions);
LLVM_ABI void clear();
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 15ea107d04625..013e1e7eeccc8 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -379,7 +379,7 @@ class SCEVPredicate : public FoldingSetNode {
FoldingSetNodeIDRef FastID;
public:
- enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap, P_TripCountInvariant };
+ enum SCEVPredicateKind { P_Union, P_Compare, P_Wrap };
protected:
SCEVPredicateKind Kind;
@@ -464,40 +464,6 @@ class LLVM_ABI SCEVComparePredicate final : public SCEVPredicate {
}
};
-/// This class extends the SCEVPredicate class to represent the assumption that
-/// the trip count of a loop is invariant. This predicate doesn't lower any
-/// numeric run-time check but solely depends on the memory alias checks
-/// generated for the loop by LoopVectorizer. NOTE: Users of this predicate make
-/// sure to verify that the memory alias checks are generated for the loop.
-class LLVM_ABI SCEVTripCountInvariantPredicate final : public SCEVPredicate {
- const SCEV *TripCountLoad;
- const SCEV *TripCountInvariantLoad;
-
-public:
- SCEVTripCountInvariantPredicate(const FoldingSetNodeIDRef ID,
- const SCEV *TripCountLoad,
- const SCEV *TripCountInvariantLoad);
-
- /// Implementation of the SCEVPredicate interface
- bool implies(const SCEVPredicate *N, ScalarEvolution &SE) const override;
- void print(raw_ostream &OS, unsigned Depth = 0) const override;
- bool isAlwaysTrue() const override;
-
- /// This predicate doesn't generate any runtime checks, and solely depends
- /// upon the memory alias checks for the runtime check generation, so
- /// contributes nothing to the check complexity
- unsigned getComplexity() const override { return 0; }
-
- const SCEV *getTripCountLoad() const { return TripCountLoad; }
- const SCEV *getTripCountInvariantLoad() const {
- return TripCountInvariantLoad;
- }
-
- static bool classof(const SCEVPredicate *P) {
- return P->getKind() == P_TripCountInvariant;
- }
-};
-
/// This class represents an assumption made on an AddRec expression. Given an
/// affine AddRec expression {a,+,b}, we assume that it has the nssw or nusw
/// flags (defined below) in the first X iterations of the loop, where X is a
@@ -1123,8 +1089,11 @@ class ScalarEvolution {
/// SCEV predicates to Predicates that are required to be true in order for
/// the answer to be correct. Predicates can be checked with run-time
/// checks and can be used to perform loop versioning.
+ /// Assumptions holds predicates which are applied to the exit conditions
+ /// before they are analyzed;
LLVM_ABI const SCEV *getPredicatedBackedgeTakenCount(
- const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates);
+ const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates,
+ const SCEVPredicate *Assumptions = nullptr);
/// When successful, this returns a SCEVConstant that is greater than or equal
/// to (i.e. a "conservative over-approximation") of the value returend by
@@ -1153,14 +1122,11 @@ class ScalarEvolution {
/// SCEV predicates to Predicates that are required to be true in order for
/// the answer to be correct. Predicates can be checked with run-time
/// checks and can be used to perform loop versioning.
+ /// Assumptions holds predicates which are applied to the exit conditions
+ /// before they are analyzed;
LLVM_ABI const SCEV *getPredicatedSymbolicMaxBackedgeTakenCount(
- const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates);
-
- /// To compute the backedge-taken count under the assumption of
- /// TripCountInvariant predicate.
- LLVM_ABI const SCEV *computeBackedgeTakenCountWithTripCountInvariants(
- const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
- bool SymbolicMax = false);
+ const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Predicates,
+ const SCEVPredicate *Assumptions = nullptr);
/// Return true if the backedge taken count is either the value returned by
/// getConstantMaxBackedgeTakenCount or zero.
@@ -1562,9 +1528,6 @@ class ScalarEvolution {
getWrapPredicate(const SCEVAddRecExpr *AR,
SCEVWrapPredicate::IncrementWrapFlags AddedFlags);
- LLVM_ABI const SCEVPredicate *
- getTripCountInvariantPredicate(const SCEV *Load, const SCEV *InvariantLoad);
-
/// Re-writes the SCEV according to the Predicates in \p A.
LLVM_ABI const SCEV *rewriteUsingPredicate(const SCEV *S, const Loop *L,
const SCEVPredicate &A);
@@ -1933,8 +1896,7 @@ class ScalarEvolution {
/// function as they are computed.
DenseMap<const Loop *, BackedgeTakenInfo> PredicatedBackedgeTakenCounts;
- DenseMap<const Value *, const SCEV *> *ActiveTripCountInvariantSubst =
- nullptr;
+ const SCEVPredicate *AssumedPreds = nullptr;
/// Loops whose backedge taken counts directly use this non-constant SCEV.
DenseMap<const SCEV *, SmallPtrSet<PointerIntPair<const Loop *, 1, bool>, 4>>
@@ -2701,17 +2663,9 @@ class PredicatedScalarEvolution {
/// Adds a new predicate.
LLVM_ABI void addPredicate(const SCEVPredicate &Pred);
- LLVM_ABI void addTripCountInvariantPredicate(const SCEV *Load,
- const SCEV *InvariantLoad);
+ LLVM_ABI void addAssumption(const SCEVPredicate *Assumption);
- LLVM_ABI bool hasTripCountInvariantPredicates() const {
- return !TripCountInvariantPreds.empty();
- }
-
- LLVM_ABI ArrayRef<const SCEVTripCountInvariantPredicate *>
- getTripCountInvariantPredicates() const {
- return TripCountInvariantPreds;
- }
+ ArrayRef<const SCEVPredicate *> getAssumptions() const { return Assumptions; }
/// Adds all predicates in \p Preds.
LLVM_ABI void addPredicates(ArrayRef<const SCEVPredicate *> Preds);
@@ -2784,10 +2738,9 @@ class PredicatedScalarEvolution {
/// The symbolic backedge taken count.
const SCEV *SymbolicMaxBackedgeCount = nullptr;
- SmallVector<const SCEVTripCountInvariantPredicate *, 2>
- TripCountInvariantPreds;
+ SmallVector<const SCEVPredicate *, 2> Assumptions;
- DenseMap<const Value *, const SCEV *> TripCountInvariantMap;
+ std::unique_ptr<SCEVUnionPredicate> UnionAssumptions;
/// The constant max trip count for the loop.
std::optional<unsigned> SmallConstantMaxTripCount;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index fe2ea5201b2d9..752958d94baa3 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -3327,19 +3327,17 @@ LLVM_ABI bool llvm::collectInvariantLoadsBoundChain(
return !BoundLoads.empty() && !HoistedDeps.empty();
}
-LoopAccessInfo::LoopAccessInfo(
- Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI,
- const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT,
- LoopInfo *LI, AssumptionCache *AC, bool AllowPartial,
- ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds)
+LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
+ const TargetTransformInfo *TTI,
+ const TargetLibraryInfo *TLI, AAResults *AA,
+ DominatorTree *DT, LoopInfo *LI,
+ AssumptionCache *AC, bool AllowPartial,
+ ArrayRef<const SCEVPredicate *> Assumptions)
: PSE(std::make_unique<PredicatedScalarEvolution>(*SE, *L)),
PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial),
- TripCountInvariantPreds(TripCountInvariantPreds) {
-
- for (const SCEVTripCountInvariantPredicate *P : TripCountInvariantPreds) {
- PSE->addTripCountInvariantPredicate(P->getTripCountLoad(),
- P->getTripCountInvariantLoad());
- }
+ Assumptions(Assumptions) {
+ for (const SCEVPredicate *P : Assumptions)
+ PSE->addAssumption(P);
unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
if (TTI && !TTI->enableScalableVectorization())
@@ -3416,28 +3414,30 @@ const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L,
return getInfo(L, AllowPartial, {});
}
-const LoopAccessInfo &LoopAccessInfoManager::getInfo(
- Loop &L, bool AllowPartial,
- ArrayRef<const SCEVTripCountInvariantPredicate *> TripCountInvariantPreds) {
+const LoopAccessInfo &
+LoopAccessInfoManager::getInfo(Loop &L, bool AllowPartial,
+ ArrayRef<const SCEVPredicate *> Assumptions) {
auto Insertion = LoopAccessInfoMap.try_emplace(&L);
auto It = Insertion.first;
bool Inserted = Insertion.second;
- auto SamePreds = [&]() {
- ArrayRef<const SCEVTripCountInvariantPredicate *> Cached =
- It->second->getTripCountInvariantPreds();
- if (Cached.size() != TripCountInvariantPreds.size())
+ auto SameAssumptions = [&]() {
+ ArrayRef<const SCEVPredicate *> CachedAssumptions =
+ It->second->getAssumptions();
+ if (CachedAssumptions.size() != Assumptions.size()) {
return false;
- return std::equal(Cached.begin(), Cached.end(),
- TripCountInvariantPreds.begin());
+ }
+ return std::equal(CachedAssumptions.begin(), CachedAssumptions.end(),
+ Assumptions.begin());
};
- // We need to create the LoopAccessInfo if either we don't already have one,
- // or if it was created with a different value of AllowPartial.
- if (Inserted || It->second->hasAllowPartial() != AllowPartial || !SamePreds())
- It->second =
- std::make_unique<LoopAccessInfo>(&L, &SE, TTI, TLI, &AA, &DT, &LI, AC,
- AllowPartial, TripCountInvariantPreds);
+ // We need to create the LoopAccessInfo if we don't already have one, or if it
+ // was created with a different value of AllowPartial, or under different
+ // assumptions.
+ if (Inserted || It->second->hasAllowPartial() != AllowPartial ||
+ !SameAssumptions())
+ It->second = std::make_unique<LoopAccessInfo>(
+ &L, &SE, TTI, TLI, &AA, &DT, &LI, AC, AllowPartial, Assumptions);
return *It->second;
}
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 02926865366cf..33ea9c885b9b3 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8649,8 +8649,15 @@ const SCEV *ScalarEvolution::getPredicatedExitCount(
}
const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
- const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
- return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds);
+ const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds,
+ const SCEVPredicate *Assumptions) {
+ if (!Assumptions || Assumptions->isAlwaysTrue())
+ return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds);
+
+ (void)getBackedgeTakenInfo(L);
+
+ SaveAndRestore<const SCEVPredicate *> Assumed(AssumedPreds, Assumptions);
+ return computeBackedgeTakenCount(L, true).getExact(L, this, &Preds);
}
const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
@@ -8667,8 +8674,16 @@ const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
}
const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
- const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
- return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, this, &Preds);
+ const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds,
+ const SCEVPredicate *Assumptions) {
+ if (!Assumptions || Assumptions->isAlwaysTrue())
+ return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, this, &Preds);
+
+ (void)getBackedgeTakenInfo(L);
+
+ SaveAndRestore<const SCEVPredicate *> Assumed(AssumedPreds, Assumptions);
+ BackedgeTakenInfo BTI = computeBackedgeTakenCount(L, true);
+ return BTI.getSymbolicMax(L, this, &Preds);
}
const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
@@ -8676,30 +8691,6 @@ const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
return getPredicatedBackedgeTakenInfo(L).getConstantMax(this, &Preds);
}
-const SCEV *ScalarEvolution::computeBackedgeTakenCountWithTripCountInvariants(
- const Loop *L, ArrayRef<const SCEVTripCountInvariantPredicate *> Preds,
- bool SymbolicMax) {
- if (Preds.empty())
- return SymbolicMax ? getSymbolicMaxBackedgeTakenCount(L)
- : getBackedgeTakenCount(L);
-
- ValueToSCEVMapTy Subst;
- for (const SCEVTripCountInvariantPredicate *P : Preds) {
- const auto *V = dyn_cast<SCEVUnknown>(P->getTripCountLoad());
- if (!V)
- return getCouldNotCompute();
- Subst[V->getValue()] = P->getTripCountInvariantLoad();
- }
-
- (void)getBackedgeTakenInfo(L);
-
- SaveAndRestore<DenseMap<const Value *, const SCEV *> *> Active(
- ActiveTripCountInvariantSubst, &Subst);
- BackedgeTakenInfo BTI =
- computeBackedgeTakenCount(L, /*AllowPredicates=*/false);
- return SymbolicMax ? BTI.getSymbolicMax(L, this) : BTI.getExact(L, this);
-}
-
bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
}
@@ -9488,15 +9479,10 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
- // If the TripCountInvariant predicate is active, then replace the LHS and RHS
- // to their proper substition values as per the substitution map.
- if (ActiveTripCountInvariantSubst &&
- !ActiveTripCountInvariantSubst->empty()) {
- auto &Subst =
- const_cast<ValueToSCEVMapTy &>(*ActiveTripCountInvariantSubst);
-
- LHS = SCEVParameterRewriter::rewrite(LHS, *this, Subst);
- RHS = SCEVParameterRewriter::rewrite(RHS, *this, Subst);
+ // Rewrite the SCEV with the equivalent SCEV mapped in predicates.
+ if (AssumedPreds) {
+ LHS = rewriteUsingPredicate(LHS, L, *AssumedPreds);
+ RHS = rewriteUsingPredicate(RHS, L, *AssumedPreds);
}
ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, ControlsOnlyExit,
@@ -15298,22 +15284,6 @@ const SCEVPredicate *ScalarEvolution::getWrapPredicate(
return OF;
}
-const SCEVPredicate *
-ScalarEvolution::getTripCountInvariantPredicate(const SCEV *Load,
- const SCEV *InvariantLoad) {
- FoldingSetNodeID ID;
- ID.AddInteger(SCEVPredicate::P_TripCountInvariant);
- ID.AddPointer(Load);
- ID.AddPointer(InvariantLoad);
- void *IP = nullptr;
- if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
- return S;
- auto *TCI = new (SCEVAllocator) SCEVTripCountInvariantPredicate(
- ID.Intern(SCEVAllocator), Load, InvariantLoad);
- UniquePreds.InsertNode(TCI, IP);
- return TCI;
-}
-
namespace {
class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
@@ -15521,32 +15491,6 @@ void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
}
-SCEVTripCountInvariantPredicate::SCEVTripCountInvariantPredicate(
- const FoldingSetNodeIDRef ID, const SCEV *TripCountLoad,
- const SCEV *TripCountInvariantLoad)
- : SCEVPredicate(ID, P_TripCountInvariant), TripCountLoad(TripCountLoad),
- TripCountInvariantLoad(TripCountInvariantLoad) {}
-
-bool SCEVTripCountInvariantPredicate::implies(const SCEVPredicate *N,
- ScalarEvolution &SE) const {
- const auto *Op = dyn_cast<SCEVTripCountInvariantPredicate>(N);
- if (!Op)
- return false;
- return Op->TripCountLoad == TripCountLoad &&
- Op->TripCountInvariantLoad == TripCountInvariantLoad;
-}
-
-bool SCEVTripCountInvariantPredicate::isAlwaysTrue() const {
- return TripCountLoad == TripCountInvariantLoad;
-}
-
-void SCEVTripCountInvariantPredicate::print(raw_ostream &OS,
- unsigned Depth) const {
- OS.indent(Depth) << "Trip count invariant predicate: " << *TripCountLoad
- << " is invariant (== " << *TripCountInvariantLoad << ")\n"
- << "\n";
-}
-
SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
const SCEVAddRecExpr *AR,
IncrementWrapFlags Flags)
@@ -15761,12 +15705,6 @@ const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
Expr = Entry.second;
const SCEV *NewSCEV = SE.rewriteUsingPredicate(Expr, &L, *Preds);
-
- if (!TripCountInvariantMap.empty()) {
- NewSCEV =
- SCEVParameterRewriter::rewrite(NewSCEV, SE, TripCountInvariantMap);
- }
-
Entry = {Generation, NewSCEV};
return NewSCEV;
@@ -15774,32 +15712,22 @@ const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
if (!BackedgeCount) {
- if (!TripCountInvariantPreds.empty()) {
- BackedgeCount = SE.computeBackedgeTakenCountWithTripCountInvariants(
- &L, TripCountInvariantPreds, /*SymbolicMax=*/false);
- } else {
- SmallVector<const SCEVPredicate *, 4> Preds;
- BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, Preds);
- for (const auto *P : Preds)
- addPredicate(*P);
- }
+ SmallVector<const SCEVPredicate *, 4> Preds;
+ BackedgeCount =
+ SE.getPredicatedBackedgeTakenCount(&L, Preds, UnionAssumptions.get());
+ for (const auto *P : Preds)
+ addPredicate(*P);
}
return BackedgeCount;
}
const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
if (!SymbolicMaxBackedgeCount) {
- if (!TripCountInvariantPreds.empty()) {
- SymbolicMaxBackedgeCount =
- SE.computeBackedgeTakenCountWithTripCountInvariants(
- &L, TripCountInvariantPreds, /*SymbolicMax=*/true);
- } else {
- SmallVector<const SCEVPredicate *, 4> Preds;
- SymbolicMaxBackedgeCount =
- SE.getPredicatedSymbolicMaxBackedgeTakenCount(&L, Preds);
- for (const auto *P : Preds)
- addPredicate(*P);
- }
+ SmallVector<const SCEVPredicate *, 4> Preds;
+ SymbolicMaxBackedgeCount = SE.getPredicatedSymbolicMaxBackedgeTakenCount(
+ &L, Preds, UnionAssumptions.get());
+ for (const auto *P : Preds)
+ addPredicate(*P);
}
return SymbolicMaxBackedgeCount;
}
@@ -15824,22 +15752,17 @@ void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
updateGeneration();
}
-void PredicatedScalarEvolution::addTripCountInvariantPredicate(
- const SCEV *Load, const SCEV *InvariantLoad) {
- const auto *V = dyn_cast<SCEVUnknown>(Load);
- assert(V && "mem-invariant Load must be a SCEVUnknown");
-
- const auto *P = cast<SCEVTripCountInvariantPredicate>(
- SE.getTripCountInvariantPredicate(Load, InvariantLoad));
+void PredicatedScalarEvolution::addAssumption(const SCEVPredicate *Assumption) {
+ if (Assumption->isAlwaysTrue() || is_contained(Assumptions, Assumption))
+ return;
- if (TripCountInvariantMap.insert({V->getValue(), InvariantLoad}).second) {
- TripCountInvariantPreds.push_back(P);
- BackedgeCount = nullptr;
- SymbolicMaxBackedgeCount = nullptr;
- SmallConstantMaxTripCount.reset();
- }
+ Assumptions.push_back(Assumption);
+ UnionAssumptions = std::make_unique<SCEVUnionPredicate>(Assumptions, SE);
- addPredicate(*P);
+ // Forcing recomputation of the backedge taken count with new assumptions
+ BackedgeCount = nullptr;
+ SymbolicMaxBackedgeCount = nullptr;
+ SmallConstantMaxTripCount.reset();
}
void PredicatedScalarEvolution::addPredicates(
@@ -15900,8 +15823,10 @@ PredicatedScalarEvolution::PredicatedScalarEvolution(
Preds(std::make_unique<SCEVUnionPredicate>(Init.Preds->getPredicates(),
SE)),
Generation(Init.Generation), BackedgeCount(Init.BackedgeCount),
- TripCountInvariantPreds(Init.TripCountInvariantPreds),
- TripCountInvariantMap(Init.TripCountInvariantMap) {}
+ Assumptions(Init.Assumptions) {
+ if (!Assumptions.empty())
+ UnionAssumptions = std::make_unique<SCEVUnionPredicate>(Assumptions, SE);
+}
void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
// For each block.
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index f46eb9aff16e3..125571b18dddc 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2211,8 +2211,6 @@ Value *SCEVExpander::expandCodeForPredicate(const SCEVPredicate *Pred,
auto *AddRecPred = cast<SCEVWrapPredicate>(Pred);
return expandWrapPredicate(AddRecPred, IP);
}
- case SCEVPredicate::P_TripCountInvariant:
- return ConstantInt::getFalse(IP->getContext());
}
llvm_unreachable("Unknown SCEV predicate type");
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index ddae25fc17386..e33191b02fe58 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1158,7 +1158,7 @@ bool LoopVectorizationLegality::canVectorizeIndirectUnsafeDependences() {
}
bool LoopVectorizationLegality::canVectorizeMemory() {
- LAI = &LAIs.getInfo(*TheLoop, false, PSE.getTripCountInvariantPredicates());
+ LAI = &LAIs.getInfo(*TheLoop, false, PSE.getAssumptions());
const OptimizationRemarkAnalysis *LAR = LAI->getReport();
if (LAR) {
ORE->emit([&]() {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 95a0cd21a7291..d551ef1b28c1a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1636,28 +1636,16 @@ class GeneratedRTChecks {
BasicBlock *LoopHeader = L->getHeader();
BasicBlock *Preheader = L->getLoopPreheader();
- SmallVector<const SCEVPredicate *, 4> SCEVPredsVec;
- if (const auto *U = dyn_cast<SCEVUnionPredicate>(&UnionPred)) {
- for (const SCEVPredicate *P : U->getPredicates()) {
- if (!isa<SCEVTripCountInvariantPredicate>(P)) {
- SCEVPredsVec.push_back(P);
- }
- }
- } else if (!isa<SCEVTripCountInvariantPredicate>(&UnionPred)) {
- SCEVPredsVec.push_back(&UnionPred);
- }
- SCEVUnionPredicate FilteredPred(SCEVPredsVec, *PSE.getSE());
-
// Use SplitBlock to create blocks for SCEV & memory runtime checks to
// ensure the blocks are properly added to LoopInfo & DominatorTree. Those
// may be used by SCEVExpander. The blocks will be un-linked from their
// predecessors and removed from LI & DT at the end of the function.
- if (!FilteredPred.isAlwaysTrue()) {
+ if (!UnionPred.isAlwaysTrue()) {
SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
nullptr, "vector.scevcheck");
SCEVCheckCond = SCEVExp.expandCodeForPredicate(
- &FilteredPred, SCEVCheckBlock->getTerminator());
+ &UnionPred, SCEVCheckBlock->getTerminator());
if (isa<Constant>(SCEVCheckCond)) {
// Clean up directly after expanding the predicate to a constant, to
// avoid further expansions re-using anything left over from SCEVExp.
@@ -7925,7 +7913,7 @@ bool EnableLoadBoundVectorization(Loop *L, PredicatedScalarEvolution &PSE,
if (!isa<SCEVUnknown>(LoadSCEV)) {
continue;
}
- PSE.addTripCountInvariantPredicate(LoadSCEV, InvSCEV);
+ PSE.addAssumption(SE->getEqualPredicate(LoadSCEV, InvSCEV));
Added = true;
}
if (!Added) {
>From 3e589ee8ddf4321709afd1a07967170fca0a5c1a Mon Sep 17 00:00:00 2001
From: Manish Srivastava <manish.srivastava at amd.com>
Date: Thu, 6 Aug 2026 13:51:40 +0530
Subject: [PATCH 5/5] Changes for LIT testcases as suggested in review
---
.../dynamic-bound-array-element.ll | 22 +++---
.../dynamic-bound-chained-loads.ll | 24 +++----
.../LoopVectorize/dynamic-bound-fcmp-exit.ll | 26 +++----
.../LoopVectorize/dynamic-bound-lambda.ll | 49 +++++--------
.../dynamic-bound-loop-varying.ll | 24 +++----
.../LoopVectorize/dynamic-bound-simple.ll | 63 +++++++----------
.../dynamic-bound-stored-pointer.ll | 28 ++++----
.../LoopVectorize/dynamic-bound-two-loads.ll | 68 +++++++------------
.../dynamic-bound-volatile-load.ll | 26 +++----
9 files changed, 142 insertions(+), 188 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
index 9de99bfede967..fc3376d7148b2 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-array-element.ll
@@ -12,30 +12,30 @@
; }
;
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
entry:
- %arrayidx = getelementptr inbounds nuw i32, ptr %A, i64 2
+ %arrayidx = getelementptr inbounds nuw i32, ptr %a, i64 2
%0 = load i32, ptr %arrayidx, align 4
%cmp11 = icmp sgt i32 %0, 0
- br i1 %cmp11, label %for.body, label %for.cond.cleanup
+ br i1 %cmp11, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx1 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx1 = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%1 = load i32, ptr %arrayidx1, align 4
- %arrayidx3 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx3 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%2 = load i32, ptr %arrayidx3, align 4
%add = add nsw i32 %2, %1
- %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx5 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add, ptr %arrayidx5, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
%3 = load i32, ptr %arrayidx, align 4
%4 = sext i32 %3 to i64
- %cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %ec = icmp slt i64 %indvars.iv.next, %4
+ br i1 %ec, label %loop, label %exit
}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
index e53cf9b9ee738..59987fea67f37 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-chained-loads.ll
@@ -10,32 +10,32 @@
; A[i] = B[i] + C[i];
; }
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %PtrLen) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %ptrlen) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- %ptr0 = load ptr, ptr %PtrLen, align 8
+ %ptr0 = load ptr, ptr %ptrlen, align 8
%val0 = load i32, ptr %ptr0, align 4
%cmp9 = icmp sgt i32 %val0, 0
- br i1 %cmp9, label %for.body, label %for.cond.cleanup
+ br i1 %cmp9, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%1 = load i32, ptr %arrayidx, align 4
- %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%2 = load i32, ptr %arrayidx2, align 4
%add = add nsw i32 %2, %1
- %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add, ptr %arrayidx4, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %ptr = load ptr, ptr %PtrLen, align 8
+ %ptr = load ptr, ptr %ptrlen, align 8
%val = load i32, ptr %ptr, align 4
%3 = sext i32 %val to i64
- %cmp = icmp slt i64 %indvars.iv.next, %3
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %ec = icmp slt i64 %indvars.iv.next, %3
+ br i1 %ec, label %loop, label %exit
}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
index b039314b24297..4437ad6562a5d 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-fcmp-exit.ll
@@ -10,26 +10,26 @@
; A[(int)x] = B[(int)x];
; }
-define void @foo(ptr %A, ptr %B, ptr %Limit) {
+define void @foo(ptr %a, ptr %b, ptr %limit) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- br label %for.body
+ br label %loop
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %fiv = phi float [ 0.0, %entry ], [ %fiv.next, %for.body ]
- %gep.b = getelementptr inbounds i32, ptr %B, i64 %iv
- %b = load i32, ptr %gep.b, align 4
- %gep.a = getelementptr inbounds i32, ptr %A, i64 %iv
- store i32 %b, ptr %gep.a, align 4
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fiv = phi float [ 0.0, %entry ], [ %fiv.next, %loop ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b.val = load i32, ptr %gep.b, align 4
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %b.val, ptr %gep.a, align 4
%iv.next = add nuw nsw i64 %iv, 1
%fiv.next = fadd float %fiv, 1.0
- %lim = load float, ptr %Limit, align 4
- %cmp = fcmp olt float %fiv.next, %lim
- br i1 %cmp, label %for.body, label %for.exit
+ %lim = load float, ptr %limit, align 4
+ %ec = fcmp olt float %fiv.next, %lim
+ br i1 %ec, label %loop, label %exit
-for.exit:
+exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
index 6f86515e1a867..06d57c71b32ad 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-lambda.ll
@@ -12,14 +12,14 @@
; }
;
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %D, ptr %len) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %d, ptr %len) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[LEN:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[CMP10_I:%.*]] = icmp sgt i32 [[TMP0]], 0
-; CHECK-NEXT: br i1 [[CMP10_I]], label %[[FOR_BODY_I_PREHEADER:.*]], label %[[EXIT:.*]]
-; CHECK: [[FOR_BODY_I_PREHEADER]]:
+; CHECK-NEXT: br i1 [[CMP10_I]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
@@ -64,10 +64,10 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %D, ptr %len) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_I_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY_I:.*]]
-; CHECK: [[FOR_BODY_I]]:
-; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_I:%.*]], %[[FOR_BODY_I]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV_I]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX_I]], align 4
; CHECK-NEXT: [[ARRAYIDX3_I:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV_I]]
@@ -78,8 +78,8 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %D, ptr %len) {
; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nuw nsw i64 [[INDVARS_IV_I]], 1
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
-; CHECK-NEXT: [[CMP_I:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_I]], [[TMP11]]
-; CHECK-NEXT: br i1 [[CMP_I]], label %[[FOR_BODY_I]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: [[EC:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT_I]], [[TMP11]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -88,38 +88,23 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %D, ptr %len) {
entry:
%0 = load i32, ptr %len, align 4
%cmp10.i = icmp sgt i32 %0, 0
- br i1 %cmp10.i, label %for.body.i, label %exit
+ br i1 %cmp10.i, label %loop, label %exit
-for.body.i:
- %indvars.iv.i = phi i64 [ %indvars.iv.next.i, %for.body.i ], [ 0, %entry ]
- %arrayidx.i = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv.i
+loop:
+ %indvars.iv.i = phi i64 [ %indvars.iv.next.i, %loop ], [ 0, %entry ]
+ %arrayidx.i = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv.i
%1 = load i32, ptr %arrayidx.i, align 4
- %arrayidx3.i = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv.i
+ %arrayidx3.i = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv.i
%2 = load i32, ptr %arrayidx3.i, align 4
%mul.i = mul nsw i32 %2, %1
- %arrayidx5.i = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv.i
+ %arrayidx5.i = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv.i
store i32 %mul.i, ptr %arrayidx5.i, align 4
%indvars.iv.next.i = add nuw nsw i64 %indvars.iv.i, 1
%3 = load i32, ptr %len, align 4
%4 = sext i32 %3 to i64
- %cmp.i = icmp slt i64 %indvars.iv.next.i, %4
- br i1 %cmp.i, label %for.body.i, label %exit
+ %ec = icmp slt i64 %indvars.iv.next.i, %4
+ br i1 %ec, label %loop, label %exit
exit:
ret void
}
-;.
-; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
-; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
-; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
-; CHECK: [[META7]] = !{[[META1]], [[META4]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
-; CHECK: [[META10]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META11]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META10]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
index 0942f94be1fde..94e2b67c737ce 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-loop-varying.ll
@@ -9,31 +9,31 @@
; A[i] = B[i] + C[i];
; }
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- %0 = load i32, ptr %Len, align 4
+ %0 = load i32, ptr %len, align 4
%cmp12 = icmp sgt i32 %0, 0
- br i1 %cmp12, label %for.body, label %for.cond.cleanup
+ br i1 %cmp12, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx2 = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%1 = load i32, ptr %arrayidx2, align 4
- %arrayidx4 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%2 = load i32, ptr %arrayidx4, align 4
%add = add nsw i32 %2, %1
- %arrayidx6 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx6 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add, ptr %arrayidx6, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %arrayidx = getelementptr inbounds nuw i32, ptr %Len, i64 %indvars.iv.next
+ %arrayidx = getelementptr inbounds nuw i32, ptr %len, i64 %indvars.iv.next
%3 = load i32, ptr %arrayidx, align 4
%4 = sext i32 %3 to i64
- %cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %ec = icmp slt i64 %indvars.iv.next, %4
+ br i1 %ec, label %loop, label %exit
}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
index 58a6e65dd5f47..c678cef1188f6 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-simple.ll
@@ -8,14 +8,14 @@
; A[i] = B[i] + C[i];
; }
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %len) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[LEN:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[TMP0]], 0
-; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br i1 [[CMP9]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = sext i32 [[DOTBOUND_PRE]] to i64
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.smax.i64(i64 [[DOTBOUND_PRE1]], i64 1)
@@ -58,16 +58,16 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
@@ -78,44 +78,29 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[LEN]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP11]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: [[EC:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP11]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP12:![0-9]+]]
;
entry:
- %0 = load i32, ptr %Len, align 4
+ %0 = load i32, ptr %len, align 4
%cmp9 = icmp sgt i32 %0, 0
- br i1 %cmp9, label %for.body, label %for.cond.cleanup
+ br i1 %cmp9, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%1 = load i32, ptr %arrayidx, align 4
- %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%2 = load i32, ptr %arrayidx2, align 4
%add = add nsw i32 %2, %1
- %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add, ptr %arrayidx4, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %3 = load i32, ptr %Len, align 4
+ %3 = load i32, ptr %len, align 4
%4 = sext i32 %3 to i64
- %cmp = icmp slt i64 %indvars.iv.next, %4
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %ec = icmp slt i64 %indvars.iv.next, %4
+ br i1 %ec, label %loop, label %exit
}
-;.
-; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
-; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
-; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
-; CHECK: [[META7]] = !{[[META1]], [[META4]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
-; CHECK: [[META10]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META11]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META10]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
index a1a0ed6ba277b..c6225b15f33da 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-stored-pointer.ll
@@ -11,31 +11,31 @@
; }
; }
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- %len0 = load i32, ptr %Len, align 4
+ %len0 = load i32, ptr %len, align 4
%cmp0 = icmp sgt i32 %len0, 0
- br i1 %cmp0, label %for.body, label %for.cond.cleanup
+ br i1 %cmp0, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%0 = load i32, ptr %arrayidx, align 4
- %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%1 = load i32, ptr %arrayidx2, align 4
%add = add nsw i32 %1, %0
- %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add, ptr %arrayidx4, align 4
- store i32 %add, ptr %Len, align 4
+ store i32 %add, ptr %len, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %len = load i32, ptr %Len, align 4
- %ext = sext i32 %len to i64
- %cmp = icmp slt i64 %indvars.iv.next, %ext
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %len.val = load i32, ptr %len, align 4
+ %ext = sext i32 %len.val to i64
+ %ec = icmp slt i64 %indvars.iv.next, %ext
+ br i1 %ec, label %loop, label %exit
}
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
index ddfb275a7dbd9..50488c3b07ba4 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-two-loads.ll
@@ -8,7 +8,7 @@
; A[i] = B[i] + C[i];
; }
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len1, ptr %Len2) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %len1, ptr %len2) {
; CHECK-LABEL: define void @foo(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[LEN1:%.*]], ptr [[LEN2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -16,8 +16,8 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len1, ptr %Len2) {
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[LEN2]], align 4
; CHECK-NEXT: [[ADD10:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[CMP11:%.*]] = icmp sgt i32 [[ADD10]], 0
-; CHECK-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; CHECK: [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT: br i1 [[CMP11]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[DOTBOUND_PRE:%.*]] = load i32, ptr [[LEN1]], align 4
; CHECK-NEXT: [[DOTBOUND_PRE1:%.*]] = load i32, ptr [[LEN2]], align 4
; CHECK-NEXT: [[ADD_BOUND_PRE:%.*]] = add nsw i32 [[DOTBOUND_PRE1]], [[DOTBOUND_PRE]]
@@ -73,16 +73,16 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len1, ptr %Len2) {
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-NEXT: br label %[[FOR_COND_CLEANUP]]
-; CHECK: [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[INDVARS_IV]]
@@ -95,49 +95,33 @@ define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len1, ptr %Len2) {
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[LEN2]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP18]], [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[ADD]] to i64
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP19]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: [[EC:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP19]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP13:![0-9]+]]
;
entry:
- %0 = load i32, ptr %Len1, align 4
- %1 = load i32, ptr %Len2, align 4
+ %0 = load i32, ptr %len1, align 4
+ %1 = load i32, ptr %len2, align 4
%add10 = add nsw i32 %1, %0
%cmp11 = icmp sgt i32 %add10, 0
- br i1 %cmp11, label %for.body, label %for.cond.cleanup
+ br i1 %cmp11, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%2 = load i32, ptr %arrayidx, align 4
- %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%3 = load i32, ptr %arrayidx2, align 4
%add3 = add nsw i32 %3, %2
- %arrayidx5 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx5 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add3, ptr %arrayidx5, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %4 = load i32, ptr %Len1, align 4
- %5 = load i32, ptr %Len2, align 4
+ %4 = load i32, ptr %len1, align 4
+ %5 = load i32, ptr %len2, align 4
%add = add nsw i32 %5, %4
%6 = sext i32 %add to i64
- %cmp = icmp slt i64 %indvars.iv.next, %6
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %ec = icmp slt i64 %indvars.iv.next, %6
+ br i1 %ec, label %loop, label %exit
}
-;.
-; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
-; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
-; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
-; CHECK: [[META7]] = !{[[META1]], [[META4]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
-; CHECK: [[META9]] = distinct !{[[META9]], [[META2]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]], [[META12:![0-9]+]]}
-; CHECK: [[META11]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META12]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META11]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll b/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
index 0441064259bf6..1d8b51a4fd29e 100644
--- a/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/dynamic-bound-volatile-load.ll
@@ -8,30 +8,30 @@
; A[i] = B[i] + C[i];
; }
-define void @foo(ptr %A, ptr %B, ptr %C, ptr %Len) {
+define void @foo(ptr %a, ptr %b, ptr %c, ptr %len) {
; CHECK-LABEL: @foo(
; CHECK-NOT: vector.body
; CHECK-NOT: .bound.pre
entry:
- %len0 = load volatile i32, ptr %Len, align 4
+ %len0 = load volatile i32, ptr %len, align 4
%cmp0 = icmp sgt i32 %len0, 0
- br i1 %cmp0, label %for.body, label %for.cond.cleanup
+ br i1 %cmp0, label %loop, label %exit
-for.cond.cleanup:
+exit:
ret void
-for.body:
- %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds nuw i32, ptr %B, i64 %indvars.iv
+loop:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds nuw i32, ptr %b, i64 %indvars.iv
%0 = load i32, ptr %arrayidx, align 4
- %arrayidx2 = getelementptr inbounds nuw i32, ptr %C, i64 %indvars.iv
+ %arrayidx2 = getelementptr inbounds nuw i32, ptr %c, i64 %indvars.iv
%1 = load i32, ptr %arrayidx2, align 4
%add = add nsw i32 %1, %0
- %arrayidx4 = getelementptr inbounds nuw i32, ptr %A, i64 %indvars.iv
+ %arrayidx4 = getelementptr inbounds nuw i32, ptr %a, i64 %indvars.iv
store i32 %add, ptr %arrayidx4, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %len = load volatile i32, ptr %Len, align 4
- %ext = sext i32 %len to i64
- %cmp = icmp slt i64 %indvars.iv.next, %ext
- br i1 %cmp, label %for.body, label %for.cond.cleanup
+ %len.val = load volatile i32, ptr %len, align 4
+ %ext = sext i32 %len.val to i64
+ %ec = icmp slt i64 %indvars.iv.next, %ext
+ br i1 %ec, label %loop, label %exit
}
More information about the llvm-commits
mailing list