[llvm] dc0d00c - Revert "[LAA/LV] Simplify stride speculation logic [NFC]"
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Thu May 11 09:26:54 PDT 2023
Author: Philip Reames
Date: 2023-05-11T09:26:35-07:00
New Revision: dc0d00c5fcac5e292f67e19f61bf0bbe436dfd03
URL: https://github.com/llvm/llvm-project/commit/dc0d00c5fcac5e292f67e19f61bf0bbe436dfd03
DIFF: https://github.com/llvm/llvm-project/commit/dc0d00c5fcac5e292f67e19f61bf0bbe436dfd03.diff
LOG: Revert "[LAA/LV] Simplify stride speculation logic [NFC]"
This reverts commit d5b840131223f2ffef4e48ca769ad1eb7bb1869a. Running this through broader testing after rebasing is revealing a crash. Reverting while I investigate.
Added:
Modified:
llvm/include/llvm/Analysis/LoopAccessAnalysis.h
llvm/include/llvm/Analysis/VectorUtils.h
llvm/lib/Analysis/LoopAccessAnalysis.cpp
llvm/lib/Analysis/VectorUtils.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 11b4d621d7640..fae22d484a09b 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -184,7 +184,7 @@ class MemoryDepChecker {
///
/// Only checks sets with elements in \p CheckDeps.
bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps,
- const DenseMap<Value *, const SCEV *> &Strides);
+ const ValueToValueMap &Strides);
/// No memory dependence was encountered that would inhibit
/// vectorization.
@@ -316,7 +316,7 @@ class MemoryDepChecker {
/// Otherwise, this function returns true signaling a possible dependence.
Dependence::DepType isDependent(const MemAccessInfo &A, unsigned AIdx,
const MemAccessInfo &B, unsigned BIdx,
- const DenseMap<Value *, const SCEV *> &Strides);
+ const ValueToValueMap &Strides);
/// Check whether the data dependence could prevent store-load
/// forwarding.
@@ -612,9 +612,7 @@ class LoopAccessInfo {
/// If an access has a symbolic strides, this maps the pointer value to
/// the stride symbol.
- const DenseMap<Value *, const SCEV *> &getSymbolicStrides() const {
- return SymbolicStrides;
- }
+ const ValueToValueMap &getSymbolicStrides() const { return SymbolicStrides; }
/// Pointer has a symbolic stride.
bool hasStride(Value *V) const { return StrideSet.count(V); }
@@ -701,7 +699,7 @@ class LoopAccessInfo {
/// If an access has a symbolic strides, this maps the pointer value to
/// the stride symbol.
- DenseMap<Value *, const SCEV *> SymbolicStrides;
+ ValueToValueMap SymbolicStrides;
/// Set of symbolic strides values.
SmallPtrSet<Value *, 8> StrideSet;
@@ -718,10 +716,9 @@ Value *stripIntegerCast(Value *V);
///
/// \p PtrToStride provides the mapping between the pointer value and its
/// stride as collected by LoopVectorizationLegality::collectStridedAccess.
-const SCEV *
-replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &PtrToStride,
- Value *Ptr);
+const SCEV *replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
+ const ValueToValueMap &PtrToStride,
+ Value *Ptr);
/// If the pointer has a constant stride return it in units of the access type
/// size. Otherwise return std::nullopt.
@@ -740,7 +737,7 @@ replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
std::optional<int64_t>
getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
const Loop *Lp,
- const DenseMap<Value *, const SCEV *> &StridesMap = DenseMap<Value *, const SCEV *>(),
+ const ValueToValueMap &StridesMap = ValueToValueMap(),
bool Assume = false, bool ShouldCheckWrap = true);
/// Returns the distance between the pointers \p PtrA and \p PtrB iff they are
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 18214fe19700e..5d824540c3896 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -917,7 +917,7 @@ class InterleavedAccessInfo {
/// Collect all the accesses with a constant stride in program order.
void collectConstStrideAccesses(
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
- const DenseMap<Value *, const SCEV *> &Strides);
+ const ValueToValueMap &Strides);
/// Returns true if \p Stride is allowed in an interleaved group.
static bool isStrided(int Stride);
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 7ac51c3c6ef03..351e09094d487 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -154,19 +154,21 @@ Value *llvm::stripIntegerCast(Value *V) {
}
const SCEV *llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &PtrToStride,
+ const ValueToValueMap &PtrToStride,
Value *Ptr) {
const SCEV *OrigSCEV = PSE.getSCEV(Ptr);
// If there is an entry in the map return the SCEV of the pointer with the
// symbolic stride replaced by one.
- DenseMap<Value *, const SCEV *>::const_iterator SI = PtrToStride.find(Ptr);
+ ValueToValueMap::const_iterator SI = PtrToStride.find(Ptr);
if (SI == PtrToStride.end())
// For a non-symbolic stride, just return the original expression.
return OrigSCEV;
+ Value *StrideVal = stripIntegerCast(SI->second);
+
ScalarEvolution *SE = PSE.getSE();
- const SCEV *StrideSCEV = SI->second;
+ const SCEV *StrideSCEV = SE->getSCEV(StrideVal);
assert(isa<SCEVUnknown>(StrideSCEV) && "shouldn't be in map");
const auto *CT = SE->getOne(StrideSCEV->getType());
@@ -656,7 +658,7 @@ class AccessAnalysis {
/// the bounds of the pointer.
bool createCheckForAccess(RuntimePointerChecking &RtCheck,
MemAccessInfo Access, Type *AccessTy,
- const DenseMap<Value *, const SCEV *> &Strides,
+ const ValueToValueMap &Strides,
DenseMap<Value *, unsigned> &DepSetId,
Loop *TheLoop, unsigned &RunningDepId,
unsigned ASId, bool ShouldCheckStride, bool Assume);
@@ -667,7 +669,7 @@ class AccessAnalysis {
/// Returns true if we need no check or if we do and we can generate them
/// (i.e. the pointers have computable bounds).
bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, ScalarEvolution *SE,
- Loop *TheLoop, const DenseMap<Value *, const SCEV *> &Strides,
+ Loop *TheLoop, const ValueToValueMap &Strides,
Value *&UncomputablePtr, bool ShouldCheckWrap = false);
/// Goes over all memory accesses, checks whether a RT check is needed
@@ -762,7 +764,7 @@ static bool hasComputableBounds(PredicatedScalarEvolution &PSE, Value *Ptr,
/// Check whether a pointer address cannot wrap.
static bool isNoWrap(PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &Strides, Value *Ptr, Type *AccessTy,
+ const ValueToValueMap &Strides, Value *Ptr, Type *AccessTy,
Loop *L) {
const SCEV *PtrScev = PSE.getSCEV(Ptr);
if (PSE.getSE()->isLoopInvariant(PtrScev, L))
@@ -955,7 +957,7 @@ static void findForkedSCEVs(
static SmallVector<PointerIntPair<const SCEV *, 1, bool>>
findForkedPointer(PredicatedScalarEvolution &PSE,
- const DenseMap<Value *, const SCEV *> &StridesMap, Value *Ptr,
+ const ValueToValueMap &StridesMap, Value *Ptr,
const Loop *L) {
ScalarEvolution *SE = PSE.getSE();
assert(SE->isSCEVable(Ptr->getType()) && "Value is not SCEVable!");
@@ -980,7 +982,7 @@ findForkedPointer(PredicatedScalarEvolution &PSE,
bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
MemAccessInfo Access, Type *AccessTy,
- const DenseMap<Value *, const SCEV *> &StridesMap,
+ const ValueToValueMap &StridesMap,
DenseMap<Value *, unsigned> &DepSetId,
Loop *TheLoop, unsigned &RunningDepId,
unsigned ASId, bool ShouldCheckWrap,
@@ -1041,7 +1043,7 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
ScalarEvolution *SE, Loop *TheLoop,
- const DenseMap<Value *, const SCEV *> &StridesMap,
+ const ValueToValueMap &StridesMap,
Value *&UncomputablePtr, bool ShouldCheckWrap) {
// Find pointers with computable bounds. We are going to use this information
// to place a runtime bound check.
@@ -1371,7 +1373,7 @@ static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR,
std::optional<int64_t> llvm::getPtrStride(PredicatedScalarEvolution &PSE,
Type *AccessTy, Value *Ptr,
const Loop *Lp,
- const DenseMap<Value *, const SCEV *> &StridesMap,
+ const ValueToValueMap &StridesMap,
bool Assume, bool ShouldCheckWrap) {
Type *Ty = Ptr->getType();
assert(Ty->isPointerTy() && "Unexpected non-ptr");
@@ -1820,7 +1822,7 @@ static bool areStridedAccessesIndependent(uint64_t Distance, uint64_t Stride,
MemoryDepChecker::Dependence::DepType
MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
const MemAccessInfo &B, unsigned BIdx,
- const DenseMap<Value *, const SCEV *> &Strides) {
+ const ValueToValueMap &Strides) {
assert (AIdx < BIdx && "Must pass arguments in program order");
auto [APtr, AIsWrite] = A;
@@ -2014,7 +2016,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
bool MemoryDepChecker::areDepsSafe(DepCandidates &AccessSets,
MemAccessInfoList &CheckDeps,
- const DenseMap<Value *, const SCEV *> &Strides) {
+ const ValueToValueMap &Strides) {
MaxSafeDepDistBytes = -1;
SmallPtrSet<MemAccessInfo, 8> Visited;
@@ -2689,12 +2691,6 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
if (!Ptr)
return;
- // Note: getStrideFromPointer is a *profitability* heuristic. We
- // could broaden the scope of values returned here - to anything
- // which happens to be loop invariant and contributes to the
- // computation of an interesting IV - but we chose not to as we
- // don't have a cost model here, and broadening the scope exposes
- // far too many unprofitable cases.
Value *Stride = getStrideFromPointer(Ptr, PSE->getSE(), TheLoop);
if (!Stride)
return;
@@ -2750,7 +2746,7 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
}
LLVM_DEBUG(dbgs() << "LAA: Found a strided access that we can version.\n");
- SymbolicStrides[Ptr] = StrideExpr;
+ SymbolicStrides[Ptr] = Stride;
StrideSet.insert(Stride);
}
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 5da21e75a20cb..423ef670bb3c2 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1011,7 +1011,7 @@ bool InterleavedAccessInfo::isStrided(int Stride) {
void InterleavedAccessInfo::collectConstStrideAccesses(
MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,
- const DenseMap<Value*, const SCEV*> &Strides) {
+ const ValueToValueMap &Strides) {
auto &DL = TheLoop->getHeader()->getModule()->getDataLayout();
// Since it's desired that the load/store instructions be maintained in
@@ -1091,7 +1091,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses(
void InterleavedAccessInfo::analyzeInterleaving(
bool EnablePredicatedInterleavedMemAccesses) {
LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n");
- const auto &Strides = LAI->getSymbolicStrides();
+ const ValueToValueMap &Strides = LAI->getSymbolicStrides();
// Holds all accesses with a constant stride.
MapVector<Instruction *, StrideDescriptor> AccessStrideInfo;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ffefb94d5dbd2..80a3f13f304b0 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3477,7 +3477,7 @@ InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
static bool containsDecreasingPointers(Loop *TheLoop,
PredicatedScalarEvolution *PSE) {
- const auto &Strides = DenseMap<Value *, const SCEV *>();
+ const ValueToValueMap &Strides = ValueToValueMap();
for (BasicBlock *BB : TheLoop->blocks()) {
// Scan the instructions in the block and look for addresses that are
// consecutive and decreasing.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index a2b5c04dfd149..3a868e8625a85 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -456,8 +456,8 @@ int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
// it's collected. This happens from canVectorizeWithIfConvert, when the
// pointer is checked to reference consecutive elements suitable for a
// masked access.
- const auto &Strides =
- LAI ? LAI->getSymbolicStrides() : DenseMap<Value *, const SCEV *>();
+ const ValueToValueMap &Strides =
+ LAI ? LAI->getSymbolicStrides() : ValueToValueMap();
Function *F = TheLoop->getHeader()->getParent();
bool OptForSize = F->hasOptSize() ||
More information about the llvm-commits
mailing list