[llvm] r346758 - [VPlan] VPlan version of InterleavedAccessInfo.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 13 07:58:19 PST 2018
Author: fhahn
Date: Tue Nov 13 07:58:18 2018
New Revision: 346758
URL: http://llvm.org/viewvc/llvm-project?rev=346758&view=rev
Log:
[VPlan] VPlan version of InterleavedAccessInfo.
This patch turns InterleaveGroup into a template with the instruction type
being a template parameter. It also adds a VPInterleavedAccessInfo class, which
only contains a mapping from VPInstructions to their respective InterleaveGroup.
As we do not have access to scalar evolution in VPlan, we can re-use
convert InterleavedAccessInfo to VPInterleavedAccess info.
Reviewers: Ayal, mssimpso, hfinkel, dcaballe, rengolin, mkuper, hsaito
Reviewed By: rengolin
Differential Revision: https://reviews.llvm.org/D49489
Modified:
llvm/trunk/include/llvm/Analysis/VectorUtils.h
llvm/trunk/lib/Analysis/VectorUtils.cpp
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/trunk/lib/Transforms/Vectorize/VPlan.cpp
llvm/trunk/lib/Transforms/Vectorize/VPlan.h
llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h
Modified: llvm/trunk/include/llvm/Analysis/VectorUtils.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Analysis/VectorUtils.h?rev=346758&r1=346757&r2=346758&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Analysis/VectorUtils.h (original)
+++ llvm/trunk/include/llvm/Analysis/VectorUtils.h Tue Nov 13 07:58:18 2018
@@ -24,7 +24,7 @@ namespace llvm {
template <typename T> class ArrayRef;
class DemandedBits;
class GetElementPtrInst;
-class InterleaveGroup;
+template <typename InstTy> class InterleaveGroup;
class Loop;
class ScalarEvolution;
class TargetTransformInfo;
@@ -138,7 +138,7 @@ Instruction *propagateMetadata(Instructi
/// create[*]Mask() utilities which create a shuffle mask (mask that
/// consists of indices).
Constant *createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
- const InterleaveGroup &Group);
+ const InterleaveGroup<Instruction> &Group);
/// Create a mask with replicated elements.
///
@@ -233,9 +233,12 @@ Value *concatenateVectors(IRBuilder<> &B
///
/// Note: the interleaved load group could have gaps (missing members), but
/// the interleaved store group doesn't allow gaps.
-class InterleaveGroup {
+template <typename InstTy> class InterleaveGroup {
public:
- InterleaveGroup(Instruction *Instr, int Stride, unsigned Align)
+ InterleaveGroup(unsigned Factor, bool Reverse, unsigned Align)
+ : Factor(Factor), Reverse(Reverse), Align(Align), InsertPos(nullptr) {}
+
+ InterleaveGroup(InstTy *Instr, int Stride, unsigned Align)
: Align(Align), InsertPos(Instr) {
assert(Align && "The alignment should be non-zero");
@@ -256,7 +259,7 @@ public:
/// negative if it is the new leader.
///
/// \returns false if the instruction doesn't belong to the group.
- bool insertMember(Instruction *Instr, int Index, unsigned NewAlign) {
+ bool insertMember(InstTy *Instr, int Index, unsigned NewAlign) {
assert(NewAlign && "The new member's alignment should be non-zero");
int Key = Index + SmallestKey;
@@ -288,7 +291,7 @@ public:
/// Get the member with the given index \p Index
///
/// \returns nullptr if contains no such member.
- Instruction *getMember(unsigned Index) const {
+ InstTy *getMember(unsigned Index) const {
int Key = SmallestKey + Index;
auto Member = Members.find(Key);
if (Member == Members.end())
@@ -299,16 +302,17 @@ public:
/// Get the index for the given member. Unlike the key in the member
/// map, the index starts from 0.
- unsigned getIndex(Instruction *Instr) const {
- for (auto I : Members)
+ unsigned getIndex(const InstTy *Instr) const {
+ for (auto I : Members) {
if (I.second == Instr)
return I.first - SmallestKey;
+ }
llvm_unreachable("InterleaveGroup contains no such member");
}
- Instruction *getInsertPos() const { return InsertPos; }
- void setInsertPos(Instruction *Inst) { InsertPos = Inst; }
+ InstTy *getInsertPos() const { return InsertPos; }
+ void setInsertPos(InstTy *Inst) { InsertPos = Inst; }
/// Add metadata (e.g. alias info) from the instructions in this group to \p
/// NewInst.
@@ -316,12 +320,7 @@ public:
/// FIXME: this function currently does not add noalias metadata a'la
/// addNewMedata. To do that we need to compute the intersection of the
/// noalias info from all members.
- void addMetadata(Instruction *NewInst) const {
- SmallVector<Value *, 4> VL;
- std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
- [](std::pair<int, Instruction *> p) { return p.second; });
- propagateMetadata(NewInst, VL);
- }
+ void addMetadata(InstTy *NewInst) const;
/// Returns true if this Group requires a scalar iteration to handle gaps.
bool requiresScalarEpilogue() const {
@@ -344,7 +343,7 @@ private:
unsigned Factor; // Interleave Factor.
bool Reverse;
unsigned Align;
- DenseMap<int, Instruction *> Members;
+ DenseMap<int, InstTy *> Members;
int SmallestKey = 0;
int LargestKey = 0;
@@ -359,7 +358,7 @@ private:
// store i32 %even
// %odd = add i32 // Def of %odd
// store i32 %odd // Insert Position
- Instruction *InsertPos;
+ InstTy *InsertPos;
};
/// Drive the analysis of interleaved memory accesses in the loop.
@@ -390,7 +389,7 @@ public:
/// formation for predicated accesses, we may be able to relax this limitation
/// in the future once we handle more complicated blocks.
void reset() {
- SmallPtrSet<InterleaveGroup *, 4> DelSet;
+ SmallPtrSet<InterleaveGroup<Instruction> *, 4> DelSet;
// Avoid releasing a pointer twice.
for (auto &I : InterleaveGroupMap)
DelSet.insert(I.second);
@@ -409,11 +408,16 @@ public:
/// Get the interleave group that \p Instr belongs to.
///
/// \returns nullptr if doesn't have such group.
- InterleaveGroup *getInterleaveGroup(Instruction *Instr) const {
- auto Group = InterleaveGroupMap.find(Instr);
- if (Group == InterleaveGroupMap.end())
- return nullptr;
- return Group->second;
+ InterleaveGroup<Instruction> *
+ getInterleaveGroup(const Instruction *Instr) const {
+ if (InterleaveGroupMap.count(Instr))
+ return InterleaveGroupMap.find(Instr)->second;
+ return nullptr;
+ }
+
+ iterator_range<SmallPtrSetIterator<llvm::InterleaveGroup<Instruction> *>>
+ getInterleaveGroups() {
+ return make_range(InterleaveGroups.begin(), InterleaveGroups.end());
}
/// Returns true if an interleaved group that may access memory
@@ -443,7 +447,9 @@ private:
bool RequiresScalarEpilogue = false;
/// Holds the relationships between the members and the interleave group.
- DenseMap<Instruction *, InterleaveGroup *> InterleaveGroupMap;
+ DenseMap<Instruction *, InterleaveGroup<Instruction> *> InterleaveGroupMap;
+
+ SmallPtrSet<InterleaveGroup<Instruction> *, 4> InterleaveGroups;
/// Holds dependences among the memory accesses in the loop. It maps a source
/// access to a set of dependent sink accesses.
@@ -476,19 +482,23 @@ private:
/// stride \p Stride and alignment \p Align.
///
/// \returns the newly created interleave group.
- InterleaveGroup *createInterleaveGroup(Instruction *Instr, int Stride,
- unsigned Align) {
- assert(!isInterleaved(Instr) && "Already in an interleaved access group");
- InterleaveGroupMap[Instr] = new InterleaveGroup(Instr, Stride, Align);
+ InterleaveGroup<Instruction> *
+ createInterleaveGroup(Instruction *Instr, int Stride, unsigned Align) {
+ assert(!InterleaveGroupMap.count(Instr) &&
+ "Already in an interleaved access group");
+ InterleaveGroupMap[Instr] =
+ new InterleaveGroup<Instruction>(Instr, Stride, Align);
+ InterleaveGroups.insert(InterleaveGroupMap[Instr]);
return InterleaveGroupMap[Instr];
}
/// Release the group and remove all the relationships.
- void releaseGroup(InterleaveGroup *Group) {
+ void releaseGroup(InterleaveGroup<Instruction> *Group) {
for (unsigned i = 0; i < Group->getFactor(); i++)
if (Instruction *Member = Group->getMember(i))
InterleaveGroupMap.erase(Member);
+ InterleaveGroups.erase(Group);
delete Group;
}
Modified: llvm/trunk/lib/Analysis/VectorUtils.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Analysis/VectorUtils.cpp?rev=346758&r1=346757&r2=346758&view=diff
==============================================================================
--- llvm/trunk/lib/Analysis/VectorUtils.cpp (original)
+++ llvm/trunk/lib/Analysis/VectorUtils.cpp Tue Nov 13 07:58:18 2018
@@ -505,8 +505,9 @@ Instruction *llvm::propagateMetadata(Ins
return Inst;
}
-Constant *llvm::createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
- const InterleaveGroup &Group) {
+Constant *
+llvm::createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
+ const InterleaveGroup<Instruction> &Group) {
// All 1's means mask is not needed.
if (Group.getNumMembers() == Group.getFactor())
return nullptr;
@@ -720,9 +721,9 @@ void InterleavedAccessInfo::analyzeInter
collectDependences();
// Holds all interleaved store groups temporarily.
- SmallSetVector<InterleaveGroup *, 4> StoreGroups;
+ SmallSetVector<InterleaveGroup<Instruction> *, 4> StoreGroups;
// Holds all interleaved load groups temporarily.
- SmallSetVector<InterleaveGroup *, 4> LoadGroups;
+ SmallSetVector<InterleaveGroup<Instruction> *, 4> LoadGroups;
// Search in bottom-up program order for pairs of accesses (A and B) that can
// form interleaved load or store groups. In the algorithm below, access A
@@ -744,7 +745,7 @@ void InterleavedAccessInfo::analyzeInter
// Initialize a group for B if it has an allowable stride. Even if we don't
// create a group for B, we continue with the bottom-up algorithm to ensure
// we don't break any of B's dependences.
- InterleaveGroup *Group = nullptr;
+ InterleaveGroup<Instruction> *Group = nullptr;
if (isStrided(DesB.Stride) &&
(!isPredicated(B->getParent()) || EnablePredicatedInterleavedMemAccesses)) {
Group = getInterleaveGroup(B);
@@ -789,7 +790,7 @@ void InterleavedAccessInfo::analyzeInter
// illegal code motion. A will then be free to form another group with
// instructions that precede it.
if (isInterleaved(A)) {
- InterleaveGroup *StoreGroup = getInterleaveGroup(A);
+ InterleaveGroup<Instruction> *StoreGroup = getInterleaveGroup(A);
StoreGroups.remove(StoreGroup);
releaseGroup(StoreGroup);
}
@@ -868,7 +869,7 @@ void InterleavedAccessInfo::analyzeInter
} // Iteration over B accesses.
// Remove interleaved store groups with gaps.
- for (InterleaveGroup *Group : StoreGroups)
+ for (auto *Group : StoreGroups)
if (Group->getNumMembers() != Group->getFactor()) {
LLVM_DEBUG(
dbgs() << "LV: Invalidate candidate interleaved store group due "
@@ -889,7 +890,7 @@ void InterleavedAccessInfo::analyzeInter
// This means that we can forcefully peel the loop in order to only have to
// check the first pointer for no-wrap. When we'll change to use Assume=true
// we'll only need at most one runtime check per interleaved group.
- for (InterleaveGroup *Group : LoadGroups) {
+ for (auto *Group : LoadGroups) {
// Case 1: A full group. Can Skip the checks; For full groups, if the wide
// load would wrap around the address space we would do a memory access at
// nullptr even without the transformation.
@@ -947,9 +948,9 @@ void InterleavedAccessInfo::invalidateGr
return;
// Avoid releasing a Group twice.
- SmallPtrSet<InterleaveGroup *, 4> DelSet;
+ SmallPtrSet<InterleaveGroup<Instruction> *, 4> DelSet;
for (auto &I : InterleaveGroupMap) {
- InterleaveGroup *Group = I.second;
+ InterleaveGroup<Instruction> *Group = I.second;
if (Group->requiresScalarEpilogue())
DelSet.insert(Group);
}
@@ -964,3 +965,16 @@ void InterleavedAccessInfo::invalidateGr
RequiresScalarEpilogue = false;
}
+
+template <>
+void InterleaveGroup<Instruction>::addMetadata(Instruction *NewInst) const {
+ SmallVector<Value *, 4> VL;
+ std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
+ [](std::pair<int, Instruction *> p) { return p.second; });
+ propagateMetadata(NewInst, VL);
+}
+
+template <typename InstT>
+void InterleaveGroup<InstT>::addMetadata(InstT *NewInst) const {
+ llvm_unreachable("addMetadata can only be used for Instruction");
+}
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=346758&r1=346757&r2=346758&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Tue Nov 13 07:58:18 2018
@@ -976,7 +976,7 @@ public:
/// Save vectorization decision \p W and \p Cost taken by the cost model for
/// interleaving group \p Grp and vector width \p VF.
- void setWideningDecision(const InterleaveGroup *Grp, unsigned VF,
+ void setWideningDecision(const InterleaveGroup<Instruction> *Grp, unsigned VF,
InstWidening W, unsigned Cost) {
assert(VF >= 2 && "Expected VF >=2");
/// Broadcast this decicion to all instructions inside the group.
@@ -1131,7 +1131,8 @@ public:
}
/// Get the interleaved access group that \p Instr belongs to.
- const InterleaveGroup *getInterleavedAccessGroup(Instruction *Instr) {
+ const InterleaveGroup<Instruction> *
+ getInterleavedAccessGroup(Instruction *Instr) {
return InterleaveInfo.getInterleaveGroup(Instr);
}
@@ -1994,7 +1995,8 @@ static bool useMaskedInterleavedAccesses
// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,
VectorParts *BlockInMask) {
- const InterleaveGroup *Group = Cost->getInterleavedAccessGroup(Instr);
+ const InterleaveGroup<Instruction> *Group =
+ Cost->getInterleavedAccessGroup(Instr);
assert(Group && "Fail to get an interleaved access group.");
// Skip if current instruction is not the insert position.
@@ -6377,7 +6379,7 @@ VPValue *VPRecipeBuilder::createBlockInM
VPInterleaveRecipe *VPRecipeBuilder::tryToInterleaveMemory(Instruction *I,
VFRange &Range,
VPlanPtr &Plan) {
- const InterleaveGroup *IG = CM.getInterleavedAccessGroup(I);
+ const InterleaveGroup<Instruction> *IG = CM.getInterleavedAccessGroup(I);
if (!IG)
return nullptr;
@@ -6793,7 +6795,8 @@ LoopVectorizationPlanner::buildVPlanWith
// I is a member of an InterleaveGroup for Range.Start. If it's an adjunct
// member of the IG, do not construct any Recipe for it.
- const InterleaveGroup *IG = CM.getInterleavedAccessGroup(Instr);
+ const InterleaveGroup<Instruction> *IG =
+ CM.getInterleavedAccessGroup(Instr);
if (IG && Instr != IG->getInsertPos() &&
Range.Start >= 2 && // Query is illegal for VF == 1
CM.getWideningDecision(Instr, Range.Start) ==
Modified: llvm/trunk/lib/Transforms/Vectorize/VPlan.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlan.cpp?rev=346758&r1=346757&r2=346758&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlan.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlan.cpp Tue Nov 13 07:58:18 2018
@@ -680,3 +680,48 @@ void VPWidenMemoryInstructionRecipe::pri
}
template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
+
+void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
+ Old2NewTy &Old2New,
+ InterleavedAccessInfo &IAI) {
+ ReversePostOrderTraversal<VPBlockBase *> RPOT(Region->getEntry());
+ for (VPBlockBase *Base : RPOT) {
+ visitBlock(Base, Old2New, IAI);
+ }
+}
+
+void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
+ InterleavedAccessInfo &IAI) {
+ if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
+ for (VPRecipeBase &VPI : *VPBB) {
+ assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions");
+ auto *VPInst = cast<VPInstruction>(&VPI);
+ auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
+ auto *IG = IAI.getInterleaveGroup(Inst);
+ if (!IG)
+ continue;
+
+ auto NewIGIter = Old2New.find(IG);
+ if (NewIGIter == Old2New.end())
+ Old2New[IG] = new InterleaveGroup<VPInstruction>(
+ IG->getFactor(), IG->isReverse(), IG->getAlignment());
+
+ if (Inst == IG->getInsertPos())
+ Old2New[IG]->setInsertPos(VPInst);
+
+ InterleaveGroupMap[VPInst] = Old2New[IG];
+ InterleaveGroupMap[VPInst]->insertMember(
+ VPInst, IG->getIndex(Inst),
+ IG->isReverse() ? (-1) * int(IG->getFactor()) : IG->getFactor());
+ }
+ } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
+ visitRegion(Region, Old2New, IAI);
+ else
+ llvm_unreachable("Unsupported kind of VPBlock.");
+}
+
+VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
+ InterleavedAccessInfo &IAI) {
+ Old2NewTy Old2New;
+ visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
+}
Modified: llvm/trunk/lib/Transforms/Vectorize/VPlan.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlan.h?rev=346758&r1=346757&r2=346758&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlan.h (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlan.h Tue Nov 13 07:58:18 2018
@@ -38,6 +38,7 @@
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/IRBuilder.h"
#include <algorithm>
#include <cassert>
@@ -52,7 +53,8 @@ class LoopVectorizationCostModel;
class BasicBlock;
class DominatorTree;
class InnerLoopVectorizer;
-class InterleaveGroup;
+template <class T> class InterleaveGroup;
+class LoopInfo;
class raw_ostream;
class Value;
class VPBasicBlock;
@@ -771,11 +773,11 @@ public:
/// or stores into one wide load/store and shuffles.
class VPInterleaveRecipe : public VPRecipeBase {
private:
- const InterleaveGroup *IG;
+ const InterleaveGroup<Instruction> *IG;
std::unique_ptr<VPUser> User;
public:
- VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Mask)
+ VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Mask)
: VPRecipeBase(VPInterleaveSC), IG(IG) {
if (Mask) // Create a VPInstruction to register as a user of the mask.
User.reset(new VPUser({Mask}));
@@ -793,7 +795,7 @@ public:
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent) const override;
- const InterleaveGroup *getInterleaveGroup() { return IG; }
+ const InterleaveGroup<Instruction> *getInterleaveGroup() { return IG; }
};
/// VPReplicateRecipe replicates a given instruction producing multiple scalar
@@ -1464,6 +1466,48 @@ public:
}
};
+class VPInterleavedAccessInfo {
+private:
+ DenseMap<VPInstruction *, InterleaveGroup<VPInstruction> *>
+ InterleaveGroupMap;
+
+ /// Type for mapping of instruction based interleave groups to VPInstruction
+ /// interleave groups
+ using Old2NewTy = DenseMap<InterleaveGroup<Instruction> *,
+ InterleaveGroup<VPInstruction> *>;
+
+ /// Recursively \p Region and populate VPlan based interleave groups based on
+ /// \p IAI.
+ void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
+ InterleavedAccessInfo &IAI);
+ /// Recursively traverse \p Block and populate VPlan based interleave groups
+ /// based on \p IAI.
+ void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
+ InterleavedAccessInfo &IAI);
+
+public:
+ VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI);
+
+ ~VPInterleavedAccessInfo() {
+ SmallPtrSet<InterleaveGroup<VPInstruction> *, 4> DelSet;
+ // Avoid releasing a pointer twice.
+ for (auto &I : InterleaveGroupMap)
+ DelSet.insert(I.second);
+ for (auto *Ptr : DelSet)
+ delete Ptr;
+ }
+
+ /// Get the interleave group that \p Instr belongs to.
+ ///
+ /// \returns nullptr if doesn't have such group.
+ InterleaveGroup<VPInstruction> *
+ getInterleaveGroup(VPInstruction *Instr) const {
+ if (InterleaveGroupMap.count(Instr))
+ return InterleaveGroupMap.find(Instr)->second;
+ return nullptr;
+ }
+};
+
} // end namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
Modified: llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h?rev=346758&r1=346757&r2=346758&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h Tue Nov 13 07:58:18 2018
@@ -40,6 +40,7 @@ class VPValue {
friend class VPBuilder;
friend class VPlanHCFGTransforms;
friend class VPBasicBlock;
+ friend class VPInterleavedAccessInfo;
private:
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
More information about the llvm-commits
mailing list