[llvm] [LAA] Give the symbolic strides map a named type (NFC) (PR #216731)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 17 06:30:30 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Guy David (guy-david)
<details>
<summary>Changes</summary>
LoopAccessInfo collects SCEVUnknown strides, collectStridedAccess casts to it when inserting, but the map was typed as holding a plain SCEV, so the invariant had to be restated at each use.
---
Full diff: https://github.com/llvm/llvm-project/pull/216731.diff
8 Files Affected:
- (modified) llvm/include/llvm/Analysis/LoopAccessAnalysis.h (+12-9)
- (modified) llvm/include/llvm/Analysis/VectorUtils.h (+1-1)
- (modified) llvm/lib/Analysis/LoopAccessAnalysis.cpp (+29-30)
- (modified) llvm/lib/Analysis/VectorUtils.cpp (+1-1)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+1-1)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp (+1-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+3-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+4-4)
``````````diff
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 392321448c895..897a85ae91ec8 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -55,6 +55,11 @@ struct VectorizerParams {
LLVM_ABI static bool HoistRuntimeChecks;
};
+/// Maps a pointer accessed with a symbolic (non-constant) stride to that
+/// stride. Users rely on the stride being loop invariant- only SCEVUnknown
+/// strides are ever collected, which is a conservative proxy for that.
+using SymbolicStrideMap = DenseMap<Value *, const SCEVUnknown *>;
+
/// Checks memory dependences among accesses to the same underlying
/// object to determine whether there vectorization is legal or not (and at
/// which vectorization factor).
@@ -188,7 +193,7 @@ class MemoryDepChecker {
MemoryDepChecker(PredicatedScalarEvolution &PSE, AssumptionCache *AC,
DominatorTree *DT, const Loop *L,
- const DenseMap<Value *, const SCEV *> &SymbolicStrides,
+ const SymbolicStrideMap &SymbolicStrides,
unsigned MaxTargetVectorWidthInBits,
std::optional<ScalarEvolution::LoopGuards> &LoopGuards)
: PSE(PSE), AC(AC), DT(DT), InnermostLoop(L),
@@ -323,7 +328,7 @@ class MemoryDepChecker {
/// Reference to map of pointer values to
/// their stride symbols, if they have a symbolic stride.
- const DenseMap<Value *, const SCEV *> &SymbolicStrides;
+ const SymbolicStrideMap &SymbolicStrides;
/// Maps access locations (ptr, read/write) to program order.
DenseMap<MemAccessInfo, std::vector<unsigned> > Accesses;
@@ -754,7 +759,7 @@ class LoopAccessInfo {
/// If an access has a symbolic strides, this maps the pointer value to
/// the stride symbol.
- const DenseMap<Value *, const SCEV *> &getSymbolicStrides() const {
+ const SymbolicStrideMap &getSymbolicStrides() const {
return SymbolicStrides;
}
@@ -862,7 +867,7 @@ class LoopAccessInfo {
/// If an access has a symbolic strides, this maps the pointer value to
/// the stride symbol.
- DenseMap<Value *, const SCEV *> SymbolicStrides;
+ SymbolicStrideMap SymbolicStrides;
};
/// Return the SCEV corresponding to a pointer with the symbolic stride
@@ -876,8 +881,7 @@ class LoopAccessInfo {
/// stride as collected by LoopVectorizationLegality::collectStridedAccess.
LLVM_ABI const SCEV *
replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &PtrToStride,
- Value *Ptr);
+ const SymbolicStrideMap &PtrToStride, Value *Ptr);
/// If \p AR is an affine AddRec for \p Lp with a constant step, return the
/// step in units of \p AccessTy's allocation size. Returns std::nullopt if the
@@ -905,8 +909,7 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
LLVM_ABI std::optional<int64_t>
getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
const Loop *Lp, const DominatorTree &DT,
- const DenseMap<Value *, const SCEV *> &StridesMap =
- DenseMap<Value *, const SCEV *>(),
+ const SymbolicStrideMap &StridesMap = SymbolicStrideMap(),
bool ShouldCheckWrap = true,
SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
@@ -916,7 +919,7 @@ getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
LLVM_ABI std::optional<int64_t>
getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
const Loop *Lp, const DominatorTree &DT,
- const DenseMap<Value *, const SCEV *> &StridesMap, bool Assume,
+ const SymbolicStrideMap &StridesMap, bool Assume,
bool ShouldCheckWrap = true);
/// Returns the distance between the pointers \p PtrA and \p PtrB iff they are
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index f70ae3ff57f88..b177d9eec2189 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -812,7 +812,7 @@ class InterleavedAccessInfo {
/// Predicates if it is not nullptr.
void collectConstStrideAccesses(
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
- const DenseMap<Value *, const SCEV *> &Strides,
+ const SymbolicStrideMap &Strides,
SmallVectorImpl<const SCEVPredicate *> *Predicates);
/// Returns true if \p Stride is allowed in an interleaved group.
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index e248b22de7d43..256cd6ad3b64b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -152,24 +152,19 @@ bool VectorizerParams::isInterleaveForced() {
return ::VectorizationInterleave.getNumOccurrences() > 0;
}
-const SCEV *llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &PtrToStride,
- Value *Ptr) {
+const SCEV *
+llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
+ const SymbolicStrideMap &PtrToStride,
+ Value *Ptr) {
const SCEV *OrigSCEV = PSE.getSCEV(Ptr);
// If there is an entry in the map return the SCEV of the pointer with the
// symbolic stride replaced by one.
- const SCEV *StrideSCEV = PtrToStride.lookup(Ptr);
+ const SCEVUnknown *StrideSCEV = PtrToStride.lookup(Ptr);
if (!StrideSCEV)
// For a non-symbolic stride, just return the original expression.
return OrigSCEV;
- // Note: This assert is both overly strong and overly weak. The actual
- // invariant here is that StrideSCEV should be loop invariant. The only
- // such invariant strides we happen to speculate right now are unknowns
- // and thus this is a reasonable proxy of the actual invariant.
- assert(isa<SCEVUnknown>(StrideSCEV) && "shouldn't be in map");
-
ScalarEvolution *SE = PSE.getSE();
const SCEV *CT = SE->getOne(StrideSCEV->getType());
PSE.addPredicate(*SE->getEqualPredicate(StrideSCEV, CT));
@@ -860,7 +855,7 @@ class AccessAnalysis {
/// the bounds of the pointer.
bool createCheckForAccess(RuntimePointerChecking &RtCheck,
MemAccessInfo Access, Type *AccessTy,
- const DenseMap<Value *, const SCEV *> &Strides,
+ const SymbolicStrideMap &Strides,
DenseMap<Value *, unsigned> &DepSetId,
Loop *TheLoop, unsigned &RunningDepId,
unsigned ASId, bool Assume);
@@ -875,7 +870,7 @@ class AccessAnalysis {
/// pointers we could analyze. \p DepChecker is used to remove unknown
/// dependences from DepCands.
bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, Loop *TheLoop,
- const DenseMap<Value *, const SCEV *> &Strides,
+ const SymbolicStrideMap &Strides,
Value *&UncomputablePtr, bool AllowPartial,
const MemoryDepChecker &DepChecker);
@@ -1279,11 +1274,12 @@ static void findForkedSCEVs(
}
}
-bool AccessAnalysis::createCheckForAccess(
- RuntimePointerChecking &RtCheck, MemAccessInfo Access, Type *AccessTy,
- const DenseMap<Value *, const SCEV *> &StridesMap,
- DenseMap<Value *, unsigned> &DepSetId, Loop *TheLoop,
- unsigned &RunningDepId, unsigned ASId, bool Assume) {
+bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
+ MemAccessInfo Access, Type *AccessTy,
+ const SymbolicStrideMap &StridesMap,
+ DenseMap<Value *, unsigned> &DepSetId,
+ Loop *TheLoop, unsigned &RunningDepId,
+ unsigned ASId, bool Assume) {
Value *Ptr = Access.getPointer();
ScalarEvolution *SE = PSE.getSE();
assert(SE->isSCEVable(Ptr->getType()) && "Value is not SCEVable!");
@@ -1364,10 +1360,11 @@ bool AccessAnalysis::createCheckForAccess(
return true;
}
-bool AccessAnalysis::canCheckPtrAtRT(
- RuntimePointerChecking &RtCheck, Loop *TheLoop,
- const DenseMap<Value *, const SCEV *> &StridesMap, Value *&UncomputablePtr,
- bool AllowPartial, const MemoryDepChecker &DepChecker) {
+bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
+ Loop *TheLoop,
+ const SymbolicStrideMap &StridesMap,
+ Value *&UncomputablePtr, bool AllowPartial,
+ const MemoryDepChecker &DepChecker) {
// Find pointers with computable bounds. We are going to use this information
// to place a runtime bound check.
bool CanDoRT = true;
@@ -1668,10 +1665,11 @@ void AccessAnalysis::buildDependenceSets() {
}
/// Check whether the access through \p Ptr has a constant stride.
-std::optional<int64_t> llvm::getPtrStride(
- PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp,
- const DominatorTree &DT, const DenseMap<Value *, const SCEV *> &StridesMap,
- bool ShouldCheckWrap, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
+std::optional<int64_t>
+llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
+ const Loop *Lp, const DominatorTree &DT,
+ const SymbolicStrideMap &StridesMap, bool ShouldCheckWrap,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates) {
const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr);
if (PSE.getSE()->isLoopInvariant(PtrScev, Lp))
return 0;
@@ -1705,11 +1703,12 @@ std::optional<int64_t> llvm::getPtrStride(
}
/// Check whether the access through \p Ptr has a constant stride.
-std::optional<int64_t>
-llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
- const Loop *Lp, const DominatorTree &DT,
- const DenseMap<Value *, const SCEV *> &StridesMap,
- bool Assume, bool ShouldCheckWrap) {
+std::optional<int64_t> llvm::getPtrStride(PredicatedScalarEvolution &PSE,
+ Type *AccessTy, Value *Ptr,
+ const Loop *Lp,
+ const DominatorTree &DT,
+ const SymbolicStrideMap &StridesMap,
+ bool Assume, bool ShouldCheckWrap) {
SmallVector<const SCEVPredicate *> Predicates;
std::optional<int64_t> Stride =
getPtrStride(PSE, AccessTy, Ptr, Lp, DT, StridesMap, ShouldCheckWrap,
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 193fb6720cf60..b817553003f6a 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1296,7 +1296,7 @@ bool InterleavedAccessInfo::isStrided(int Stride) {
void InterleavedAccessInfo::collectConstStrideAccesses(
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
- const DenseMap<Value *, const SCEV *> &Strides,
+ const SymbolicStrideMap &Strides,
SmallVectorImpl<const SCEVPredicate *> *Predicates) {
auto &DL = TheLoop->getHeader()->getDataLayout();
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 21a9e752cad24..62968bce6cef1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -7209,7 +7209,7 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
static bool containsDecreasingPointers(Loop *TheLoop,
PredicatedScalarEvolution *PSE,
const DominatorTree &DT) {
- const auto &Strides = DenseMap<Value *, const SCEV *>();
+ const auto &Strides = SymbolicStrideMap();
for (BasicBlock *BB : TheLoop->blocks()) {
// Scan the instructions in the block and look for addresses that are
// consecutive and decreasing.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 88f7fddc56c2c..ce5c83936a751 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -461,7 +461,7 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
// the symbolic strides when runtime SCEV checks are permitted.
const auto &Strides = LAI && AllowRuntimeSCEVChecks
? LAI->getSymbolicStrides()
- : DenseMap<Value *, const SCEV *>();
+ : SymbolicStrideMap();
SmallVector<const SCEVPredicate *> Predicates;
int Stride = getPtrStride(PSE, AccessTy, Ptr, TheLoop, *DT, Strides, false,
AllowRuntimeSCEVChecks ? &Predicates : nullptr)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5f125aecb005a..0fdec9b6fd4b9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2663,8 +2663,7 @@ void VPlanTransforms::optimize(VPlan &Plan) {
void VPlanTransforms::replaceSymbolicStrides(
VPlan &Plan, PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &StridesMap,
- const VPDominatorTree &VPDT) {
+ const SymbolicStrideMap &StridesMap, const VPDominatorTree &VPDT) {
// Replace VPValues for known constant strides guaranteed by predicated scalar
// evolution that are guaranteed to be guarded by the runtime checks; that is,
// blocks dominated by the vector header.
@@ -2681,9 +2680,9 @@ void VPlanTransforms::replaceSymbolicStrides(
return VPDT.dominates(Header, R->getParent());
};
ValueToSCEVMapTy RewriteMap;
- for (const SCEV *Stride : StridesMap.values()) {
+ for (const SCEVUnknown *Stride : StridesMap.values()) {
using namespace SCEVPatternMatch;
- auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
+ auto *StrideV = Stride->getValue();
const APInt *StrideConst;
if (!match(PSE.getSCEV(StrideV), m_scev_APInt(StrideConst)))
// Only handle constant strides for now.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index af8f6de26e0f9..a7a52cd0d5706 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -307,10 +307,10 @@ struct VPlanTransforms {
/// Replace symbolic strides from \p StridesMap in \p Plan with constants when
/// possible.
- static void
- replaceSymbolicStrides(VPlan &Plan, PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &StridesMap,
- const VPDominatorTree &VPDT);
+ static void replaceSymbolicStrides(VPlan &Plan,
+ PredicatedScalarEvolution &PSE,
+ const SymbolicStrideMap &StridesMap,
+ const VPDominatorTree &VPDT);
/// Drop poison flags from recipes that may generate a poison value that is
/// used after vectorization, even when their operands are not poison. Those
``````````
</details>
https://github.com/llvm/llvm-project/pull/216731
More information about the llvm-commits
mailing list