[llvm] r272970 - [LV] Move management of symbolic strides to LAA. NFCI
Adam Nemet via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 16 15:57:55 PDT 2016
Author: anemet
Date: Thu Jun 16 17:57:55 2016
New Revision: 272970
URL: http://llvm.org/viewvc/llvm-project?rev=272970&view=rev
Log:
[LV] Move management of symbolic strides to LAA. NFCI
This is still NFCI, so the list of clients that allow symbolic stride
speculation does not change (yes: LV and LoopVersioningLICM, no: LLE,
LDist). However since the symbolic strides are now managed by LAA
rather than passed by client a new bool parameter is used to enable
symbolic stride speculation.
The existing test Transforms/LoopVectorize/version-mem-access.ll checks
that stride speculation is performed for LV.
The previously added test Transforms/LoopLoadElim/symbolic-stride.ll
ensures that no speculation is performed for LLE.
The next patch will change the functionality and turn on symbolic stride
speculation in all of LAA's clients and remove the bool parameter.
Modified:
llvm/trunk/include/llvm/Analysis/LoopAccessAnalysis.h
llvm/trunk/lib/Analysis/LoopAccessAnalysis.cpp
llvm/trunk/lib/Transforms/Scalar/LoopVersioningLICM.cpp
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
Modified: llvm/trunk/include/llvm/Analysis/LoopAccessAnalysis.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Analysis/LoopAccessAnalysis.h?rev=272970&r1=272969&r2=272970&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Analysis/LoopAccessAnalysis.h (original)
+++ llvm/trunk/include/llvm/Analysis/LoopAccessAnalysis.h Thu Jun 16 17:57:55 2016
@@ -514,7 +514,7 @@ public:
LoopAccessInfo(Loop *L, ScalarEvolution *SE, const DataLayout &DL,
const TargetLibraryInfo *TLI, AliasAnalysis *AA,
DominatorTree *DT, LoopInfo *LI,
- const ValueToValueMap &Strides);
+ bool SpeculateSymbolicStrides);
/// Return true we can analyze the memory accesses in the loop and there are
/// no memory dependence cycles.
@@ -575,13 +575,20 @@ public:
return DepChecker.getInstructionsForAccess(Ptr, isWrite);
}
+ /// \brief If an access has a symbolic strides, this maps the pointer value to
+ /// the stride symbol.
+ const ValueToValueMap &getSymbolicStrides() const { return SymbolicStrides; }
+
+ /// \brief Pointer has a symbolic stride.
+ bool hasStride(Value *V) const { return StrideSet.count(V); }
+
/// \brief Print the information about the memory accesses in the loop.
void print(raw_ostream &OS, unsigned Depth = 0) const;
/// \brief Used to ensure that if the analysis was run with speculating the
/// value of symbolic strides, the client queries it with the same assumption.
/// Only used in DEBUG build but we don't want NDEBUG-dependent ABI.
- unsigned NumSymbolicStrides;
+ bool SpeculateSymbolicStrides;
/// \brief Checks existence of store to invariant address inside loop.
/// If the loop has any store to invariant address, then it returns true,
@@ -598,8 +605,8 @@ public:
PredicatedScalarEvolution PSE;
private:
- /// \brief Analyze the loop. Substitute symbolic strides using Strides.
- void analyzeLoop(const ValueToValueMap &SymbolicStrides);
+ /// \brief Analyze the loop.
+ void analyzeLoop();
/// \brief Check if the structure of the loop allows it to be analyzed by this
/// pass.
@@ -607,6 +614,12 @@ private:
void emitAnalysis(LoopAccessReport &Message);
+ /// \brief Collect memory access with loop invariant strides.
+ ///
+ /// Looks for accesses like "a[i * StrideA]" where "StrideA" is loop
+ /// invariant.
+ void collectStridedAccess(Value *LoadOrStoreInst);
+
/// We need to check that all of the pointers in this list are disjoint
/// at runtime.
RuntimePointerChecking PtrRtChecking;
@@ -637,6 +650,13 @@ private:
/// \brief The diagnostics report generated for the analysis. E.g. why we
/// couldn't analyze the loop.
Optional<LoopAccessReport> Report;
+
+ /// \brief If an access has a symbolic strides, this maps the pointer value to
+ /// the stride symbol.
+ ValueToValueMap SymbolicStrides;
+
+ /// \brief Set of symbolic strides values.
+ SmallPtrSet<Value *, 8> StrideSet;
};
Value *stripIntegerCast(Value *V);
@@ -695,12 +715,11 @@ public:
/// \brief Query the result of the loop access information for the loop \p L.
///
- /// If the client speculates (and then issues run-time checks) for the values
- /// of symbolic strides, \p Strides provides the mapping (see
- /// replaceSymbolicStrideSCEV). If there is no cached result available run
- /// the analysis.
- const LoopAccessInfo &
- getInfo(Loop *L, const ValueToValueMap &Strides = ValueToValueMap());
+ /// \p SpeculateSymbolicStrides enables symbolic value speculation. The
+ /// corresponding run-time checks are collected in LAI::PSE.
+ ///
+ /// If there is no cached result available run the analysis.
+ const LoopAccessInfo &getInfo(Loop *L, bool SpeculateSymbolicStrides = false);
void releaseMemory() override {
// Invalidate the cache when the pass is freed.
Modified: llvm/trunk/lib/Analysis/LoopAccessAnalysis.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Analysis/LoopAccessAnalysis.cpp?rev=272970&r1=272969&r2=272970&view=diff
==============================================================================
--- llvm/trunk/lib/Analysis/LoopAccessAnalysis.cpp (original)
+++ llvm/trunk/lib/Analysis/LoopAccessAnalysis.cpp Thu Jun 16 17:57:55 2016
@@ -1488,8 +1488,7 @@ bool LoopAccessInfo::canAnalyzeLoop() {
return true;
}
-void LoopAccessInfo::analyzeLoop(const ValueToValueMap &SymbolicStrides) {
-
+void LoopAccessInfo::analyzeLoop() {
typedef SmallPtrSet<Value*, 16> ValueSet;
// Holds the Load and Store instructions.
@@ -1541,6 +1540,8 @@ void LoopAccessInfo::analyzeLoop(const V
NumLoads++;
Loads.push_back(Ld);
DepChecker.addAccess(Ld);
+ if (SpeculateSymbolicStrides)
+ collectStridedAccess(Ld);
continue;
}
@@ -1563,6 +1564,8 @@ void LoopAccessInfo::analyzeLoop(const V
NumStores++;
Stores.push_back(St);
DepChecker.addAccess(St);
+ if (SpeculateSymbolicStrides)
+ collectStridedAccess(St);
}
} // Next instr.
} // Next block.
@@ -1879,17 +1882,37 @@ LoopAccessInfo::addRuntimeChecks(Instruc
return addRuntimeChecks(Loc, PtrRtChecking.getChecks());
}
+void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
+ Value *Ptr = nullptr;
+ if (LoadInst *LI = dyn_cast<LoadInst>(MemAccess))
+ Ptr = LI->getPointerOperand();
+ else if (StoreInst *SI = dyn_cast<StoreInst>(MemAccess))
+ Ptr = SI->getPointerOperand();
+ else
+ return;
+
+ Value *Stride = getStrideFromPointer(Ptr, PSE.getSE(), TheLoop);
+ if (!Stride)
+ return;
+
+ DEBUG(dbgs() << "LAA: Found a strided access that we can version");
+ DEBUG(dbgs() << " Ptr: " << *Ptr << " Stride: " << *Stride << "\n");
+ SymbolicStrides[Ptr] = Stride;
+ StrideSet.insert(Stride);
+}
+
LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
const DataLayout &DL,
const TargetLibraryInfo *TLI, AliasAnalysis *AA,
DominatorTree *DT, LoopInfo *LI,
- const ValueToValueMap &Strides)
- : PSE(*SE, *L), PtrRtChecking(SE), DepChecker(PSE, L), TheLoop(L), DL(DL),
- TLI(TLI), AA(AA), DT(DT), LI(LI), NumLoads(0), NumStores(0),
+ bool SpeculateSymbolicStrides)
+ : SpeculateSymbolicStrides(SpeculateSymbolicStrides), PSE(*SE, *L),
+ PtrRtChecking(SE), DepChecker(PSE, L), TheLoop(L), DL(DL), TLI(TLI),
+ AA(AA), DT(DT), LI(LI), NumLoads(0), NumStores(0),
MaxSafeDepDistBytes(-1U), CanVecMem(false),
StoreToLoopInvariantAddress(false) {
if (canAnalyzeLoop())
- analyzeLoop(Strides);
+ analyzeLoop();
}
void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
@@ -1933,21 +1956,18 @@ void LoopAccessInfo::print(raw_ostream &
}
const LoopAccessInfo &
-LoopAccessAnalysis::getInfo(Loop *L, const ValueToValueMap &Strides) {
+LoopAccessAnalysis::getInfo(Loop *L, bool SpeculateSymbolicStrides) {
auto &LAI = LoopAccessInfoMap[L];
#ifndef NDEBUG
- assert((!LAI || LAI->NumSymbolicStrides == Strides.size()) &&
+ assert((!LAI || LAI->SpeculateSymbolicStrides == SpeculateSymbolicStrides) &&
"Symbolic strides changed for loop");
#endif
if (!LAI) {
const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
- LAI =
- llvm::make_unique<LoopAccessInfo>(L, SE, DL, TLI, AA, DT, LI, Strides);
-#ifndef NDEBUG
- LAI->NumSymbolicStrides = Strides.size();
-#endif
+ LAI = llvm::make_unique<LoopAccessInfo>(L, SE, DL, TLI, AA, DT, LI,
+ SpeculateSymbolicStrides);
}
return *LAI.get();
}
Modified: llvm/trunk/lib/Transforms/Scalar/LoopVersioningLICM.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Scalar/LoopVersioningLICM.cpp?rev=272970&r1=272969&r2=272970&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Scalar/LoopVersioningLICM.cpp (original)
+++ llvm/trunk/lib/Transforms/Scalar/LoopVersioningLICM.cpp Thu Jun 16 17:57:55 2016
@@ -197,7 +197,6 @@ struct LoopVersioningLICM : public LoopP
bool legalLoopStructure();
bool legalLoopInstructions();
bool legalLoopMemoryAccesses();
- void collectStridedAccess(Value *LoadOrStoreInst);
bool isLoopAlreadyVisited();
void setNoAliasToLoop(Loop *);
bool instructionSafeForVersioning(Instruction *);
@@ -205,25 +204,6 @@ struct LoopVersioningLICM : public LoopP
};
}
-/// \brief Collects stride access from a given value.
-void LoopVersioningLICM::collectStridedAccess(Value *MemAccess) {
- Value *Ptr = nullptr;
- if (LoadInst *LI = dyn_cast<LoadInst>(MemAccess))
- Ptr = LI->getPointerOperand();
- else if (StoreInst *SI = dyn_cast<StoreInst>(MemAccess))
- Ptr = SI->getPointerOperand();
- else
- return;
-
- Value *Stride = getStrideFromPointer(Ptr, SE, CurLoop);
- if (!Stride)
- return;
-
- DEBUG(dbgs() << "Found a strided access that we can version");
- DEBUG(dbgs() << " Ptr: " << *Ptr << " Stride: " << *Stride << "\n");
- Strides[Ptr] = Stride;
-}
-
/// \brief Check loop structure and confirms it's good for LoopVersioningLICM.
bool LoopVersioningLICM::legalLoopStructure() {
// Loop must have a preheader, if not return false.
@@ -365,7 +345,6 @@ bool LoopVersioningLICM::instructionSafe
return false;
}
LoadAndStoreCounter++;
- collectStridedAccess(Ld);
Value *Ptr = Ld->getPointerOperand();
// Check loop invariant.
if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
@@ -380,7 +359,6 @@ bool LoopVersioningLICM::instructionSafe
return false;
}
LoadAndStoreCounter++;
- collectStridedAccess(St);
Value *Ptr = St->getPointerOperand();
// Check loop invariant.
if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
@@ -407,7 +385,7 @@ bool LoopVersioningLICM::legalLoopInstru
return false;
}
// Get LoopAccessInfo from current loop.
- LAI = &LAA->getInfo(CurLoop, Strides);
+ LAI = &LAA->getInfo(CurLoop, true);
// Check LoopAccessInfo for need of runtime check.
if (LAI->getRuntimePointerChecking()->getChecks().empty()) {
DEBUG(dbgs() << " LAA: Runtime check not found !!\n");
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=272970&r1=272969&r2=272970&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Thu Jun 16 17:57:55 2016
@@ -1374,7 +1374,7 @@ public:
unsigned getMaxSafeDepDistBytes() { return LAI->getMaxSafeDepDistBytes(); }
- bool hasStride(Value *V) { return StrideSet.count(V); }
+ bool hasStride(Value *V) { return LAI->hasStride(V); }
/// Returns true if the target machine supports masked store operation
/// for the given \p DataType and kind of access to \p Ptr.
@@ -1428,12 +1428,6 @@ private:
/// and we know that we can read from them without segfault.
bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs);
- /// \brief Collect memory access with loop invariant strides.
- ///
- /// Looks for accesses like "a[i * StrideA]" where "StrideA" is loop
- /// invariant.
- void collectStridedAccess(Value *LoadOrStoreInst);
-
/// Updates the vectorization state by adding \p Phi to the inductions list.
/// This can set \p Phi as the main induction of the loop if \p Phi is a
/// better choice for the main induction than the existing one.
@@ -1450,7 +1444,13 @@ private:
/// \brief If an access has a symbolic strides, this maps the pointer value to
/// the stride symbol.
- const ValueToValueMap *getSymbolicStrides() { return &SymbolicStrides; }
+ const ValueToValueMap *getSymbolicStrides() {
+ // FIXME: Currently, the set of symbolic strides is sometimes queried before
+ // it's collected. This happens from canVectorizeWithIfConvert, when the
+ // pointer is checked to reference consecutive elements suitable for a
+ // masked access.
+ return LAI ? &LAI->getSymbolicStrides() : nullptr;
+ }
unsigned NumPredStores;
@@ -1512,9 +1512,6 @@ private:
/// Used to emit an analysis of any legality issues.
LoopVectorizeHints *Hints;
- ValueToValueMap SymbolicStrides;
- SmallPtrSet<Value *, 8> StrideSet;
-
/// While vectorizing these instructions we have to generate a
/// call to the appropriate masked intrinsic
SmallPtrSet<const Instruction *, 8> MaskedOp;
@@ -4894,12 +4891,6 @@ bool LoopVectorizationLegality::canVecto
<< "store instruction cannot be vectorized");
return false;
}
- if (EnableMemAccessVersioning)
- collectStridedAccess(ST);
-
- } else if (LoadInst *LI = dyn_cast<LoadInst>(it)) {
- if (EnableMemAccessVersioning)
- collectStridedAccess(LI);
// FP instructions can allow unsafe algebra, thus vectorizable by
// non-IEEE-754 compliant SIMD units.
@@ -4941,25 +4932,6 @@ bool LoopVectorizationLegality::canVecto
return true;
}
-void LoopVectorizationLegality::collectStridedAccess(Value *MemAccess) {
- Value *Ptr = nullptr;
- if (LoadInst *LI = dyn_cast<LoadInst>(MemAccess))
- Ptr = LI->getPointerOperand();
- else if (StoreInst *SI = dyn_cast<StoreInst>(MemAccess))
- Ptr = SI->getPointerOperand();
- else
- return;
-
- Value *Stride = getStrideFromPointer(Ptr, PSE.getSE(), TheLoop);
- if (!Stride)
- return;
-
- DEBUG(dbgs() << "LV: Found a strided access that we can version");
- DEBUG(dbgs() << " Ptr: " << *Ptr << " Stride: " << *Stride << "\n");
- SymbolicStrides[Ptr] = Stride;
- StrideSet.insert(Stride);
-}
-
void LoopVectorizationLegality::collectLoopUniforms() {
// We now know that the loop is vectorizable!
// Collect variables that will remain uniform after vectorization.
@@ -4998,7 +4970,7 @@ void LoopVectorizationLegality::collectL
}
bool LoopVectorizationLegality::canVectorizeMemory() {
- LAI = &LAA->getInfo(TheLoop, *getSymbolicStrides());
+ LAI = &LAA->getInfo(TheLoop, EnableMemAccessVersioning);
auto &OptionalReport = LAI->getReport();
if (OptionalReport)
emitAnalysis(VectorizationReport(*OptionalReport));
More information about the llvm-commits
mailing list