[llvm] r332654 - [LV][VPlan] Build plain CFG with simple VPInstructions for outer loops.
Diego Caballero via llvm-commits
llvm-commits at lists.llvm.org
Thu May 17 12:24:47 PDT 2018
Author: dcaballe
Date: Thu May 17 12:24:47 2018
New Revision: 332654
URL: http://llvm.org/viewvc/llvm-project?rev=332654&view=rev
Log:
[LV][VPlan] Build plain CFG with simple VPInstructions for outer loops.
Patch #3 from VPlan Outer Loop Vectorization Patch Series #1
(RFC: http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html).
Expected to be NFC for the current inner loop vectorization path. It
introduces the basic algorithm to build the VPlan plain CFG (single-level
CFG, no hierarchical CFG (H-CFG), yet) in the VPlan-native vectorization
path using VPInstructions. It includes:
- VPlanHCFGBuilder: Main class to build the VPlan H-CFG (plain CFG without nested regions, for now).
- VPlanVerifier: Main class with utilities to check the consistency of a H-CFG.
- VPlanBlockUtils: Main class with utilities to manipulate VPBlockBases in VPlan.
Reviewers: rengolin, fhahn, mkuper, mssimpso, a.elovikov, hfinkel, aprantl.
Differential Revision: https://reviews.llvm.org/D44338
Added:
llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp (with props)
llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h (with props)
llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp (with props)
llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h (with props)
llvm/trunk/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
Modified:
llvm/trunk/lib/Transforms/Vectorize/CMakeLists.txt
llvm/trunk/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/trunk/lib/Transforms/Vectorize/VPlan.h
llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h
Modified: llvm/trunk/lib/Transforms/Vectorize/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/CMakeLists.txt?rev=332654&r1=332653&r2=332654&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/CMakeLists.txt (original)
+++ llvm/trunk/lib/Transforms/Vectorize/CMakeLists.txt Thu May 17 12:24:47 2018
@@ -5,6 +5,8 @@ add_llvm_library(LLVMVectorize
SLPVectorizer.cpp
Vectorize.cpp
VPlan.cpp
+ VPlanHCFGBuilder.cpp
+ VPlanVerifier.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorizationPlanner.h?rev=332654&r1=332653&r2=332654&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorizationPlanner.h (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorizationPlanner.h Thu May 17 12:24:47 2018
@@ -39,23 +39,94 @@ private:
VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
VPInstruction *createInstruction(unsigned Opcode,
- std::initializer_list<VPValue *> Operands) {
+ ArrayRef<VPValue *> Operands) {
VPInstruction *Instr = new VPInstruction(Opcode, Operands);
- BB->insert(Instr, InsertPt);
+ if (BB)
+ BB->insert(Instr, InsertPt);
return Instr;
}
+ VPInstruction *createInstruction(unsigned Opcode,
+ std::initializer_list<VPValue *> Operands) {
+ return createInstruction(Opcode, ArrayRef<VPValue *>(Operands));
+ }
+
public:
VPBuilder() {}
- /// This specifies that created VPInstructions should be appended to
- /// the end of the specified block.
+ /// Clear the insertion point: created instructions will not be inserted into
+ /// a block.
+ void clearInsertionPoint() {
+ BB = nullptr;
+ InsertPt = VPBasicBlock::iterator();
+ }
+
+ VPBasicBlock *getInsertBlock() const { return BB; }
+ VPBasicBlock::iterator getInsertPoint() const { return InsertPt; }
+
+ /// InsertPoint - A saved insertion point.
+ class VPInsertPoint {
+ VPBasicBlock *Block = nullptr;
+ VPBasicBlock::iterator Point;
+
+ public:
+ /// Creates a new insertion point which doesn't point to anything.
+ VPInsertPoint() = default;
+
+ /// Creates a new insertion point at the given location.
+ VPInsertPoint(VPBasicBlock *InsertBlock, VPBasicBlock::iterator InsertPoint)
+ : Block(InsertBlock), Point(InsertPoint) {}
+
+ /// Returns true if this insert point is set.
+ bool isSet() const { return Block != nullptr; }
+
+ VPBasicBlock *getBlock() const { return Block; }
+ VPBasicBlock::iterator getPoint() const { return Point; }
+ };
+
+ /// Sets the current insert point to a previously-saved location.
+ void restoreIP(VPInsertPoint IP) {
+ if (IP.isSet())
+ setInsertPoint(IP.getBlock(), IP.getPoint());
+ else
+ clearInsertionPoint();
+ }
+
+ /// This specifies that created VPInstructions should be appended to the end
+ /// of the specified block.
void setInsertPoint(VPBasicBlock *TheBB) {
assert(TheBB && "Attempting to set a null insert point");
BB = TheBB;
InsertPt = BB->end();
}
+ /// This specifies that created instructions should be inserted at the
+ /// specified point.
+ void setInsertPoint(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {
+ BB = TheBB;
+ InsertPt = IP;
+ }
+
+ /// Insert and return the specified instruction.
+ VPInstruction *insert(VPInstruction *I) const {
+ BB->insert(I, InsertPt);
+ return I;
+ }
+
+ /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
+ /// its underlying Instruction.
+ VPValue *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ Instruction *Inst = nullptr) {
+ VPInstruction *NewVPInst = createInstruction(Opcode, Operands);
+ NewVPInst->setUnderlyingValue(Inst);
+ return NewVPInst;
+ }
+ VPValue *createNaryOp(unsigned Opcode,
+ std::initializer_list<VPValue *> Operands,
+ Instruction *Inst = nullptr) {
+ return createNaryOp(Opcode, ArrayRef<VPValue *>(Operands), Inst);
+ }
+
VPValue *createNot(VPValue *Operand) {
return createInstruction(VPInstruction::Not, {Operand});
}
@@ -67,8 +138,28 @@ public:
VPValue *createOr(VPValue *LHS, VPValue *RHS) {
return createInstruction(Instruction::BinaryOps::Or, {LHS, RHS});
}
-};
+ //===--------------------------------------------------------------------===//
+ // RAII helpers.
+ //===--------------------------------------------------------------------===//
+
+ /// RAII object that stores the current insertion point and restores it when
+ /// the object is destroyed.
+ class InsertPointGuard {
+ VPBuilder &Builder;
+ VPBasicBlock *Block;
+ VPBasicBlock::iterator Point;
+
+ public:
+ InsertPointGuard(VPBuilder &B)
+ : Builder(B), Block(B.getInsertBlock()), Point(B.getInsertPoint()) {}
+
+ InsertPointGuard(const InsertPointGuard &) = delete;
+ InsertPointGuard &operator=(const InsertPointGuard &) = delete;
+
+ ~InsertPointGuard() { Builder.restoreIP(VPInsertPoint(Block, Point)); }
+ };
+};
/// TODO: The following VectorizationFactor was pulled out of
/// LoopVectorizationCostModel class. LV also deals with
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=332654&r1=332653&r2=332654&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Thu May 17 12:24:47 2018
@@ -56,6 +56,7 @@
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
+#include "VPlanHCFGBuilder.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
@@ -244,6 +245,17 @@ static cl::opt<bool> EnableVPlanNativePa
cl::desc("Enable VPlan-native vectorization path with "
"support for outer loop vectorization."));
+// This flag enables the stress testing of the VPlan H-CFG construction in the
+// VPlan-native vectorization path. It must be used in conjuction with
+// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
+// verification of the H-CFGs built.
+static cl::opt<bool> VPlanBuildStressTest(
+ "vplan-build-stress-test", cl::init(false), cl::Hidden,
+ cl::desc(
+ "Build VPlan for every supported loop nest in the function and bail "
+ "out right after the build (stress test the VPlan H-CFG construction "
+ "in the VPlan-native vectorization path)."));
+
/// A helper function for converting Scalar types to vector types.
/// If the incoming type is void, we return void. If the VF is 1, we return
/// the scalar type.
@@ -1653,8 +1665,11 @@ static void collectSupportedLoops(Loop &
OptimizationRemarkEmitter *ORE,
SmallVectorImpl<Loop *> &V) {
// Collect inner loops and outer loops without irreducible control flow. For
- // now, only collect outer loops that have explicit vectorization hints.
- if (L.empty() || (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
+ // now, only collect outer loops that have explicit vectorization hints. If we
+ // are stress testing the VPlan H-CFG construction, we collect the outermost
+ // loop of every loop nest.
+ if (L.empty() || VPlanBuildStressTest ||
+ (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
LoopBlocksRPO RPOT(&L);
RPOT.perform(LI);
if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
@@ -6254,7 +6269,7 @@ void LoopVectorizationCostModel::collect
VectorizationFactor
LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,
unsigned UserVF) {
- // Width 1 means no vectorize, cost 0 means uncomputed cost.
+ // Width 1 means no vectorization, cost 0 means uncomputed cost.
const VectorizationFactor NoVectorization = {1U, 0U};
// Outer loop handling: They may require CFG and instruction level
@@ -6262,12 +6277,22 @@ LoopVectorizationPlanner::planInVPlanNat
// Since we cannot modify the incoming IR, we need to build VPlan upfront in
// the vectorization pipeline.
if (!OrigLoop->empty()) {
+ // TODO: If UserVF is not provided, we set UserVF to 4 for stress testing.
+ // This won't be necessary when UserVF is not required in the VPlan-native
+ // path.
+ if (VPlanBuildStressTest && !UserVF)
+ UserVF = 4;
+
assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
assert(UserVF && "Expected UserVF for outer loop vectorization.");
assert(isPowerOf2_32(UserVF) && "VF needs to be a power of two");
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
buildVPlans(UserVF, UserVF);
+ // For VPlan build stress testing, we bail out after VPlan construction.
+ if (VPlanBuildStressTest)
+ return NoVectorization;
+
return {UserVF, 0};
}
@@ -6280,7 +6305,7 @@ LoopVectorizationPlanner::planInVPlanNat
VectorizationFactor
LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {
assert(OrigLoop->empty() && "Inner loop expected.");
- // Width 1 means no vectorize, cost 0 means uncomputed cost.
+ // Width 1 means no vectorization, cost 0 means uncomputed cost.
const VectorizationFactor NoVectorization = {1U, 0U};
Optional<unsigned> MaybeMaxVF = CM.computeMaxVF(OptForSize);
if (!MaybeMaxVF.hasValue()) // Cases considered too costly to vectorize.
@@ -6806,9 +6831,11 @@ VPBasicBlock *LoopVectorizationPlanner::
"VPBB has successors when handling predicated replication.");
// Record predicated instructions for above packing optimizations.
PredInst2Recipe[I] = Recipe;
- VPBlockBase *Region =
- VPBB->setOneSuccessor(createReplicateRegion(I, Recipe, Plan));
- return cast<VPBasicBlock>(Region->setOneSuccessor(new VPBasicBlock()));
+ VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan);
+ VPBlockUtils::insertBlockAfter(Region, VPBB);
+ auto *RegSucc = new VPBasicBlock();
+ VPBlockUtils::insertBlockAfter(RegSucc, Region);
+ return RegSucc;
}
VPRegionBlock *
@@ -6834,8 +6861,8 @@ LoopVectorizationPlanner::createReplicat
// Note: first set Entry as region entry and then connect successors starting
// from it in order, to propagate the "parent" of each VPBasicBlock.
- Entry->setTwoSuccessors(Pred, Exit);
- Pred->setOneSuccessor(Exit);
+ VPBlockUtils::insertTwoBlocksAfter(Pred, Exit, Entry);
+ VPBlockUtils::connectBlocks(Pred, Exit);
return Region;
}
@@ -6852,6 +6879,11 @@ LoopVectorizationPlanner::buildVPlan(VFR
// Create new empty VPlan
auto Plan = llvm::make_unique<VPlan>();
+
+ // Build hierarchical CFG
+ VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI);
+ HCFGBuilder.buildHierarchicalCFG(*Plan.get());
+
return Plan;
}
@@ -6893,7 +6925,7 @@ LoopVectorizationPlanner::buildVPlan(VFR
// ingredients and fill a new VPBasicBlock.
unsigned VPBBsForBB = 0;
auto *FirstVPBBForBB = new VPBasicBlock(BB->getName());
- VPBB->setOneSuccessor(FirstVPBBForBB);
+ VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
VPBB = FirstVPBBForBB;
Builder.setInsertPoint(VPBB);
@@ -6997,7 +7029,7 @@ LoopVectorizationPlanner::buildVPlan(VFR
VPBasicBlock *PreEntry = cast<VPBasicBlock>(Plan->getEntry());
assert(PreEntry->empty() && "Expecting empty pre-entry block.");
VPBlockBase *Entry = Plan->setEntry(PreEntry->getSingleSuccessor());
- PreEntry->disconnectSuccessor(Entry);
+ VPBlockUtils::disconnectBlocks(PreEntry, Entry);
delete PreEntry;
std::string PlanName;
Modified: llvm/trunk/lib/Transforms/Vectorize/VPlan.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlan.h?rev=332654&r1=332653&r2=332654&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlan.h (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlan.h Thu May 17 12:24:47 2018
@@ -306,6 +306,8 @@ struct VPTransformState {
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
class VPBlockBase {
+ friend class VPBlockUtils;
+
private:
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
@@ -372,6 +374,7 @@ public:
/// for any other purpose, as the values may change as LLVM evolves.
unsigned getVPBlockID() const { return SubclassID; }
+ VPRegionBlock *getParent() { return Parent; }
const VPRegionBlock *getParent() const { return Parent; }
void setParent(VPRegionBlock *P) { Parent = P; }
@@ -406,6 +409,9 @@ public:
return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
}
+ size_t getNumSuccessors() const { return Successors.size(); }
+ size_t getNumPredecessors() const { return Predecessors.size(); }
+
/// An Enclosing Block of a block B is any block containing B, including B
/// itself. \return the closest enclosing block starting from "this", which
/// has successors. \return the root enclosing block if all enclosing blocks
@@ -449,34 +455,31 @@ public:
return getEnclosingBlockWithPredecessors()->getSinglePredecessor();
}
- /// Sets a given VPBlockBase \p Successor as the single successor and \return
- /// \p Successor. The parent of this Block is copied to be the parent of
- /// \p Successor.
- VPBlockBase *setOneSuccessor(VPBlockBase *Successor) {
+ /// Set a given VPBlockBase \p Successor as the single successor of this
+ /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
+ /// This VPBlockBase must have no successors.
+ void setOneSuccessor(VPBlockBase *Successor) {
assert(Successors.empty() && "Setting one successor when others exist.");
appendSuccessor(Successor);
- Successor->appendPredecessor(this);
- Successor->Parent = Parent;
- return Successor;
}
- /// Sets two given VPBlockBases \p IfTrue and \p IfFalse to be the two
- /// successors. The parent of this Block is copied to be the parent of both
- /// \p IfTrue and \p IfFalse.
+ /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
+ /// successors of this VPBlockBase. This VPBlockBase is not added as
+ /// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
+ /// successors.
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
assert(Successors.empty() && "Setting two successors when others exist.");
appendSuccessor(IfTrue);
appendSuccessor(IfFalse);
- IfTrue->appendPredecessor(this);
- IfFalse->appendPredecessor(this);
- IfTrue->Parent = Parent;
- IfFalse->Parent = Parent;
}
- void disconnectSuccessor(VPBlockBase *Successor) {
- assert(Successor && "Successor to disconnect is null.");
- removeSuccessor(Successor);
- Successor->removePredecessor(this);
+ /// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
+ /// This VPBlockBase must have no predecessors. This VPBlockBase is not added
+ /// as successor of any VPBasicBlock in \p NewPreds.
+ void setPredecessors(ArrayRef<VPBlockBase *> NewPreds) {
+ assert(Predecessors.empty() && "Block predecessors already set.");
+ for (auto *Pred : NewPreds)
+ appendPredecessor(Pred);
}
/// The method which generates the output IR that correspond to this
@@ -554,10 +557,13 @@ private:
void generateInstruction(VPTransformState &State, unsigned Part);
public:
- VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands)
: VPUser(VPValue::VPInstructionSC, Operands),
VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {}
+ VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
+ : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {}
+
/// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPValue *V) {
return V->getVPValueID() == VPValue::VPInstructionSC;
@@ -963,6 +969,9 @@ public:
Entry->setParent(this);
Exit->setParent(this);
}
+ VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
+ : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exit(nullptr),
+ IsReplicator(IsReplicator) {}
~VPRegionBlock() override {
if (Entry)
@@ -977,9 +986,27 @@ public:
const VPBlockBase *getEntry() const { return Entry; }
VPBlockBase *getEntry() { return Entry; }
+ /// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
+ /// EntryBlock must have no predecessors.
+ void setEntry(VPBlockBase *EntryBlock) {
+ assert(EntryBlock->getPredecessors().empty() &&
+ "Entry block cannot have predecessors.");
+ Entry = EntryBlock;
+ EntryBlock->setParent(this);
+ }
+
const VPBlockBase *getExit() const { return Exit; }
VPBlockBase *getExit() { return Exit; }
+ /// Set \p ExitBlock as the exit VPBlockBase of this VPRegionBlock. \p
+ /// ExitBlock must have no successors.
+ void setExit(VPBlockBase *ExitBlock) {
+ assert(ExitBlock->getSuccessors().empty() &&
+ "Exit block cannot have successors.");
+ Exit = ExitBlock;
+ ExitBlock->setParent(this);
+ }
+
/// An indicator whether this region is to generate multiple replicated
/// instances of output IR corresponding to its VPBlockBases.
bool isReplicator() const { return IsReplicator; }
@@ -1007,6 +1034,13 @@ private:
/// Holds the name of the VPlan, for printing.
std::string Name;
+ /// Holds all the external definitions created for this VPlan.
+ // TODO: Introduce a specific representation for external definitions in
+ // VPlan. External definitions must be immutable and hold a pointer to its
+ // underlying IR that will be used to implement its structural comparison
+ // (operators '==' and '<').
+ SmallSet<VPValue *, 16> VPExternalDefs;
+
/// Holds a mapping between Values and their corresponding VPValue inside
/// VPlan.
Value2VPValueTy Value2VPValue;
@@ -1019,6 +1053,8 @@ public:
VPBlockBase::deleteCFG(Entry);
for (auto &MapEntry : Value2VPValue)
delete MapEntry.second;
+ for (VPValue *Def : VPExternalDefs)
+ delete Def;
}
/// Generate the IR code for this VPlan.
@@ -1037,6 +1073,12 @@ public:
void setName(const Twine &newName) { Name = newName.str(); }
+ /// Add \p VPVal to the pool of external definitions if it's not already
+ /// in the pool.
+ void addExternalDef(VPValue *VPVal) {
+ VPExternalDefs.insert(VPVal);
+ }
+
void addVPValue(Value *V) {
assert(V && "Trying to add a null Value to VPlan");
assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
@@ -1184,6 +1226,70 @@ template <> struct GraphTraits<Inverse<V
}
};
+//===----------------------------------------------------------------------===//
+// VPlan Utilities
+//===----------------------------------------------------------------------===//
+
+/// Class that provides utilities for VPBlockBases in VPlan.
+class VPBlockUtils {
+public:
+ VPBlockUtils() = delete;
+
+ /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
+ /// NewBlock as successor of \p BlockPtr and \p Block as predecessor of \p
+ /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p NewBlock
+ /// must have neither successors nor predecessors.
+ static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
+ assert(NewBlock->getSuccessors().empty() &&
+ "Can't insert new block with successors.");
+ // TODO: move successors from BlockPtr to NewBlock when this functionality
+ // is necessary. For now, setBlockSingleSuccessor will assert if BlockPtr
+ // already has successors.
+ BlockPtr->setOneSuccessor(NewBlock);
+ NewBlock->setPredecessors({BlockPtr});
+ NewBlock->setParent(BlockPtr->getParent());
+ }
+
+ /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
+ /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
+ /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
+ /// parent to \p IfTrue and \p IfFalse. \p BlockPtr must have no successors
+ /// and \p IfTrue and \p IfFalse must have neither successors nor
+ /// predecessors.
+ static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
+ VPBlockBase *BlockPtr) {
+ assert(IfTrue->getSuccessors().empty() &&
+ "Can't insert IfTrue with successors.");
+ assert(IfFalse->getSuccessors().empty() &&
+ "Can't insert IfFalse with successors.");
+ BlockPtr->setTwoSuccessors(IfTrue, IfFalse);
+ IfTrue->setPredecessors({BlockPtr});
+ IfFalse->setPredecessors({BlockPtr});
+ IfTrue->setParent(BlockPtr->getParent());
+ IfFalse->setParent(BlockPtr->getParent());
+ }
+
+ /// Connect VPBlockBases \p From and \p To bi-directionally. Append \p To to
+ /// the successors of \p From and \p From to the predecessors of \p To. Both
+ /// VPBlockBases must have the same parent, which can be null. Both
+ /// VPBlockBases can be already connected to other VPBlockBases.
+ static void connectBlocks(VPBlockBase *From, VPBlockBase *To) {
+ assert((From->getParent() == To->getParent()) &&
+ "Can't connect two block with different parents");
+ assert(From->getNumSuccessors() < 2 &&
+ "Blocks can't have more than two successors.");
+ From->appendSuccessor(To);
+ To->appendPredecessor(From);
+ }
+
+ /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
+ /// from the successors of \p From and \p From from the predecessors of \p To.
+ static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To) {
+ assert(To && "Successor to disconnect is null.");
+ From->removeSuccessor(To);
+ To->removePredecessor(From);
+ }
+};
} // end namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
Added: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp?rev=332654&view=auto
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp (added)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp Thu May 17 12:24:47 2018
@@ -0,0 +1,320 @@
+//===-- VPlanHCFGBuilder.cpp ----------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the construction of a VPlan-based Hierarchical CFG
+/// (H-CFG) for an incoming IR. This construction comprises the following
+/// components and steps:
+//
+/// 1. PlainCFGBuilder class: builds a plain VPBasicBlock-based CFG that
+/// faithfully represents the CFG in the incoming IR. A VPRegionBlock (Top
+/// Region) is created to enclose and serve as parent of all the VPBasicBlocks
+/// in the plain CFG.
+/// NOTE: At this point, there is a direct correspondence between all the
+/// VPBasicBlocks created for the initial plain CFG and the incoming
+/// BasicBlocks. However, this might change in the future.
+///
+//===----------------------------------------------------------------------===//
+
+#include "VPlanHCFGBuilder.h"
+#include "LoopVectorizationPlanner.h"
+#include "llvm/Analysis/LoopIterator.h"
+
+#define DEBUG_TYPE "loop-vectorize"
+
+using namespace llvm;
+
+// Class that is used to build the plain CFG for the incoming IR.
+class PlainCFGBuilder {
+private:
+ // The outermost loop of the input loop nest considered for vectorization.
+ Loop *TheLoop;
+
+ // Loop Info analysis.
+ LoopInfo *LI;
+
+ // Vectorization plan that we are working on.
+ VPlan &Plan;
+
+ // Output Top Region.
+ VPRegionBlock *TopRegion = nullptr;
+
+ // Builder of the VPlan instruction-level representation.
+ VPBuilder VPIRBuilder;
+
+ // NOTE: The following maps are intentionally destroyed after the plain CFG
+ // construction because subsequent VPlan-to-VPlan transformation may
+ // invalidate them.
+ // Map incoming BasicBlocks to their newly-created VPBasicBlocks.
+ DenseMap<BasicBlock *, VPBasicBlock *> BB2VPBB;
+ // Map incoming Value definitions to their newly-created VPValues.
+ DenseMap<Value *, VPValue *> IRDef2VPValue;
+
+ // Hold phi node's that need to be fixed once the plain CFG has been built.
+ SmallVector<PHINode *, 8> PhisToFix;
+
+ // Utility functions.
+ void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
+ void fixPhiNodes();
+ VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
+ bool isExternalDef(Value *Val);
+ VPValue *getOrCreateVPOperand(Value *IRVal);
+ void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
+
+public:
+ PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
+ : TheLoop(Lp), LI(LI), Plan(P) {}
+
+ // Build the plain CFG and return its Top Region.
+ VPRegionBlock *buildPlainCFG();
+};
+
+// Return true if \p Inst is an incoming Instruction to be ignored in the VPlan
+// representation.
+static bool isInstructionToIgnore(Instruction *Inst) {
+ return isa<BranchInst>(Inst);
+}
+
+// Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB
+// must have no predecessors.
+void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
+ SmallVector<VPBlockBase *, 8> VPBBPreds;
+ // Collect VPBB predecessors.
+ for (BasicBlock *Pred : predecessors(BB))
+ VPBBPreds.push_back(getOrCreateVPBB(Pred));
+
+ VPBB->setPredecessors(VPBBPreds);
+}
+
+// Add operands to VPInstructions representing phi nodes from the input IR.
+void PlainCFGBuilder::fixPhiNodes() {
+ for (auto *Phi : PhisToFix) {
+ assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
+ VPValue *VPVal = IRDef2VPValue[Phi];
+ assert(isa<VPInstruction>(VPVal) && "Expected VPInstruction for phi node.");
+ auto *VPPhi = cast<VPInstruction>(VPVal);
+ assert(VPPhi->getNumOperands() == 0 &&
+ "Expected VPInstruction with no operands.");
+
+ for (Value *Op : Phi->operands())
+ VPPhi->addOperand(getOrCreateVPOperand(Op));
+ }
+}
+
+// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
+// existing one if it was already created.
+VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
+ auto BlockIt = BB2VPBB.find(BB);
+ if (BlockIt != BB2VPBB.end())
+ // Retrieve existing VPBB.
+ return BlockIt->second;
+
+ // Create new VPBB.
+ DEBUG(dbgs() << "Creating VPBasicBlock for " << BB->getName() << "\n");
+ VPBasicBlock *VPBB = new VPBasicBlock(BB->getName());
+ BB2VPBB[BB] = VPBB;
+ VPBB->setParent(TopRegion);
+ return VPBB;
+}
+
+// Return true if \p Val is considered an external definition. An external
+// definition is either:
+// 1. A Value that is not an Instruction. This will be refined in the future.
+// 2. An Instruction that is outside of the CFG snippet represented in VPlan,
+// i.e., is not part of: a) the loop nest, b) outermost loop PH and, c)
+// outermost loop exits.
+bool PlainCFGBuilder::isExternalDef(Value *Val) {
+ // All the Values that are not Instructions are considered external
+ // definitions for now.
+ Instruction *Inst = dyn_cast<Instruction>(Val);
+ if (!Inst)
+ return true;
+
+ BasicBlock *InstParent = Inst->getParent();
+ assert(InstParent && "Expected instruction parent.");
+
+ // Check whether Instruction definition is in loop PH.
+ BasicBlock *PH = TheLoop->getLoopPreheader();
+ assert(PH && "Expected loop pre-header.");
+
+ if (InstParent == PH)
+ // Instruction definition is in outermost loop PH.
+ return false;
+
+ // Check whether Instruction definition is in the loop exit.
+ BasicBlock *Exit = TheLoop->getUniqueExitBlock();
+ assert(Exit && "Expected loop with single exit.");
+ if (InstParent == Exit) {
+ // Instruction definition is in outermost loop exit.
+ return false;
+ }
+
+ // Check whether Instruction definition is in loop body.
+ return !TheLoop->contains(Inst);
+}
+
+// Create a new VPValue or retrieve an existing one for the Instruction's
+// operand \p IRVal. This function must only be used to create/retrieve VPValues
+// for *Instruction's operands* and not to create regular VPInstruction's. For
+// the latter, please, look at 'createVPInstructionsForVPBB'.
+VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
+ auto VPValIt = IRDef2VPValue.find(IRVal);
+ if (VPValIt != IRDef2VPValue.end())
+ // Operand has an associated VPInstruction or VPValue that was previously
+ // created.
+ return VPValIt->second;
+
+ // Operand doesn't have a previously created VPInstruction/VPValue. This
+ // means that operand is:
+ // A) a definition external to VPlan,
+ // B) any other Value without specific representation in VPlan.
+ // For now, we use VPValue to represent A and B and classify both as external
+ // definitions. We may introduce specific VPValue subclasses for them in the
+ // future.
+ assert(isExternalDef(IRVal) && "Expected external definition as operand.");
+
+ // A and B: Create VPValue and add it to the pool of external definitions and
+ // to the Value->VPValue map.
+ VPValue *NewVPVal = new VPValue(IRVal);
+ Plan.addExternalDef(NewVPVal);
+ IRDef2VPValue[IRVal] = NewVPVal;
+ return NewVPVal;
+}
+
+// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
+// counterpart. This function must be invoked in RPO so that the operands of a
+// VPInstruction in \p BB have been visited before (except for Phi nodes).
+void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
+ BasicBlock *BB) {
+ VPIRBuilder.setInsertPoint(VPBB);
+ for (Instruction &InstRef : *BB) {
+ Instruction *Inst = &InstRef;
+ if (isInstructionToIgnore(Inst))
+ continue;
+
+ // There should't be any VPValue for Inst at this point. Otherwise, we
+ // visited Inst when we shouldn't, breaking the RPO traversal order.
+ assert(!IRDef2VPValue.count(Inst) &&
+ "Instruction shouldn't have been visited.");
+
+ VPInstruction *NewVPInst;
+ if (PHINode *Phi = dyn_cast<PHINode>(Inst)) {
+ // Phi node's operands may have not been visited at this point. We create
+ // an empty VPInstruction that we will fix once the whole plain CFG has
+ // been built.
+ NewVPInst = cast<VPInstruction>(VPIRBuilder.createNaryOp(
+ Inst->getOpcode(), {} /*No operands*/, Inst));
+ PhisToFix.push_back(Phi);
+ } else {
+ // Translate LLVM-IR operands into VPValue operands and set them in the
+ // new VPInstruction.
+ SmallVector<VPValue *, 4> VPOperands;
+ for (Value *Op : Inst->operands())
+ VPOperands.push_back(getOrCreateVPOperand(Op));
+
+ // Build VPInstruction for any arbitraty Instruction without specific
+ // representation in VPlan.
+ NewVPInst = cast<VPInstruction>(
+ VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
+ }
+
+ IRDef2VPValue[Inst] = NewVPInst;
+ }
+}
+
+// Main interface to build the plain CFG.
+VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
+ // 1. Create the Top Region. It will be the parent of all VPBBs.
+ TopRegion = new VPRegionBlock("TopRegion", false /*isReplicator*/);
+
+ // 2. Scan the body of the loop in a topological order to visit each basic
+ // block after having visited its predecessor basic blocks. Create a VPBB for
+ // each BB and link it to its successor and predecessor VPBBs. Note that
+ // predecessors must be set in the same order as they are in the incomming IR.
+ // Otherwise, there might be problems with existing phi nodes and algorithm
+ // based on predecessors traversal.
+
+ // Loop PH needs to be explicitly visited since it's not taken into account by
+ // LoopBlocksDFS.
+ BasicBlock *PreheaderBB = TheLoop->getLoopPreheader();
+ assert((PreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
+ "Unexpected loop preheader");
+ VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(PreheaderBB);
+ createVPInstructionsForVPBB(PreheaderVPBB, PreheaderBB);
+ // Create empty VPBB for Loop H so that we can link PH->H.
+ VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader());
+ // Preheader's predecessors will be set during the loop RPO traversal below.
+ PreheaderVPBB->setOneSuccessor(HeaderVPBB);
+
+ LoopBlocksRPO RPO(TheLoop);
+ RPO.perform(LI);
+
+ for (BasicBlock *BB : RPO) {
+ // Create or retrieve the VPBasicBlock for this BB and create its
+ // VPInstructions.
+ VPBasicBlock *VPBB = getOrCreateVPBB(BB);
+ createVPInstructionsForVPBB(VPBB, BB);
+
+ // Set VPBB successors. We create empty VPBBs for successors if they don't
+ // exist already. Recipes will be created when the successor is visited
+ // during the RPO traversal.
+ TerminatorInst *TI = BB->getTerminator();
+ assert(TI && "Terminator expected.");
+ unsigned NumSuccs = TI->getNumSuccessors();
+
+ if (NumSuccs == 1) {
+ VPBasicBlock *SuccVPBB = getOrCreateVPBB(TI->getSuccessor(0));
+ assert(SuccVPBB && "VPBB Successor not found.");
+ VPBB->setOneSuccessor(SuccVPBB);
+ } else if (NumSuccs == 2) {
+ VPBasicBlock *SuccVPBB0 = getOrCreateVPBB(TI->getSuccessor(0));
+ assert(SuccVPBB0 && "Successor 0 not found.");
+ VPBasicBlock *SuccVPBB1 = getOrCreateVPBB(TI->getSuccessor(1));
+ assert(SuccVPBB1 && "Successor 1 not found.");
+ VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1);
+ } else
+ llvm_unreachable("Number of successors not supported.");
+
+ // Set VPBB predecessors in the same order as they are in the incoming BB.
+ setVPBBPredsFromBB(VPBB, BB);
+ }
+
+ // 3. Process outermost loop exit. We created an empty VPBB for the loop
+ // single exit BB during the RPO traversal of the loop body but Instructions
+ // weren't visited because it's not part of the the loop.
+ BasicBlock *LoopExitBB = TheLoop->getUniqueExitBlock();
+ assert(LoopExitBB && "Loops with multiple exits are not supported.");
+ VPBasicBlock *LoopExitVPBB = BB2VPBB[LoopExitBB];
+ createVPInstructionsForVPBB(LoopExitVPBB, LoopExitBB);
+ // Loop exit was already set as successor of the loop exiting BB.
+ // We only set its predecessor VPBB now.
+ setVPBBPredsFromBB(LoopExitVPBB, LoopExitBB);
+
+ // 4. The whole CFG has been built at this point so all the input Values must
+ // have a VPlan couterpart. Fix VPlan phi nodes by adding their corresponding
+ // VPlan operands.
+ fixPhiNodes();
+
+ // 5. Final Top Region setup. Set outermost loop pre-header and single exit as
+ // Top Region entry and exit.
+ TopRegion->setEntry(PreheaderVPBB);
+ TopRegion->setExit(LoopExitVPBB);
+ return TopRegion;
+}
+
+// Public interface to build a H-CFG.
+void VPlanHCFGBuilder::buildHierarchicalCFG(VPlan &Plan) {
+ // Build Top Region enclosing the plain CFG and set it as VPlan entry.
+ PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
+ VPRegionBlock *TopRegion = PCFGBuilder.buildPlainCFG();
+ Plan.setEntry(TopRegion);
+ DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan);
+
+ Verifier.verifyHierarchicalCFG(TopRegion);
+}
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
------------------------------------------------------------------------------
svn:keywords = Author Date Id Rev URL
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h?rev=332654&view=auto
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h (added)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h Thu May 17 12:24:47 2018
@@ -0,0 +1,55 @@
+//===-- VPlanHCFGBuilder.h --------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines the VPlanHCFGBuilder class which contains the public
+/// interface (buildHierarchicalCFG) to build a VPlan-based Hierarchical CFG
+/// (H-CFG) for an incoming IR.
+///
+/// A H-CFG in VPlan is a control-flow graph whose nodes are VPBasicBlocks
+/// and/or VPRegionBlocks (i.e., other H-CFGs). The outermost H-CFG of a VPlan
+/// consists of a VPRegionBlock, denoted Top Region, which encloses any other
+/// VPBlockBase in the H-CFG. This guarantees that any VPBlockBase in the H-CFG
+/// other than the Top Region will have a parent VPRegionBlock and allows us
+/// to easily add more nodes before/after the main vector loop (such as the
+/// reduction epilogue).
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
+#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
+
+#include "VPlan.h"
+#include "VPlanVerifier.h"
+
+namespace llvm {
+
+class Loop;
+
+/// Main class to build the VPlan H-CFG for an incoming IR.
+class VPlanHCFGBuilder {
+private:
+ // The outermost loop of the input loop nest considered for vectorization.
+ Loop *TheLoop;
+
+ // Loop Info analysis.
+ LoopInfo *LI;
+
+ // VPlan verifier utility.
+ VPlanVerifier Verifier;
+
+public:
+ VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI) : TheLoop(Lp), LI(LI) {}
+
+ /// Build H-CFG for TheLoop and update \p Plan accordingly.
+ void buildHierarchicalCFG(VPlan &Plan);
+};
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
------------------------------------------------------------------------------
svn:keywords = Author Date Id Rev URL
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h?rev=332654&r1=332653&r2=332654&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlanValue.h Thu May 17 12:24:47 2018
@@ -37,13 +37,34 @@ class VPUser;
// coming from the input IR, instructions which VPlan will generate if executed
// and live-outs which the VPlan will need to fix accordingly.
class VPValue {
+ friend class VPBuilder;
private:
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
SmallVector<VPUser *, 1> Users;
protected:
- VPValue(const unsigned char SC) : SubclassID(SC) {}
+ // Hold the underlying Value, if any, attached to this VPValue.
+ Value *UnderlyingVal;
+
+ VPValue(const unsigned char SC, Value *UV = nullptr)
+ : SubclassID(SC), UnderlyingVal(UV) {}
+
+ // DESIGN PRINCIPLE: Access to the underlying IR must be strictly limited to
+ // the front-end and back-end of VPlan so that the middle-end is as
+ // independent as possible of the underlying IR. We grant access to the
+ // underlying IR using friendship. In that way, we should be able to use VPlan
+ // for multiple underlying IRs (Polly?) by providing a new VPlan front-end,
+ // back-end and analysis information for the new IR.
+
+ /// Return the underlying Value attached to this VPValue.
+ Value *getUnderlyingValue() { return UnderlyingVal; }
+
+ // Set \p Val as the underlying Value of this VPValue.
+ void setUnderlyingValue(Value *Val) {
+ assert(!UnderlyingVal && "Underlying Value is already set.");
+ UnderlyingVal = Val;
+ }
public:
/// An enumeration for keeping track of the concrete subclass of VPValue that
@@ -52,7 +73,7 @@ public:
/// type identification.
enum { VPValueSC, VPUserSC, VPInstructionSC };
- VPValue() : SubclassID(VPValueSC) {}
+ VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {}
VPValue(const VPValue &) = delete;
VPValue &operator=(const VPValue &) = delete;
@@ -94,11 +115,6 @@ class VPUser : public VPValue {
private:
SmallVector<VPValue *, 2> Operands;
- void addOperand(VPValue *Operand) {
- Operands.push_back(Operand);
- Operand->addUser(*this);
- }
-
protected:
VPUser(const unsigned char SC) : VPValue(SC) {}
VPUser(const unsigned char SC, ArrayRef<VPValue *> Operands) : VPValue(SC) {
@@ -120,6 +136,11 @@ public:
V->getVPValueID() <= VPInstructionSC;
}
+ void addOperand(VPValue *Operand) {
+ Operands.push_back(Operand);
+ Operand->addUser(*this);
+ }
+
unsigned getNumOperands() const { return Operands.size(); }
inline VPValue *getOperand(unsigned N) const {
assert(N < Operands.size() && "Operand index out of bounds");
Added: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp?rev=332654&view=auto
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp (added)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp Thu May 17 12:24:47 2018
@@ -0,0 +1,125 @@
+//===-- VPlanVerifier.cpp -------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines the class VPlanVerifier, which contains utility functions
+/// to check the consistency and invariants of a VPlan.
+///
+//===----------------------------------------------------------------------===//
+
+#include "VPlanVerifier.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+
+#define DEBUG_TYPE "loop-vectorize"
+
+using namespace llvm;
+
+static cl::opt<bool> EnableHCFGVerifier("vplan-verify-hcfg", cl::init(false),
+ cl::Hidden,
+ cl::desc("Verify VPlan H-CFG."));
+
+/// Utility function that checks whether \p VPBlockVec has duplicate
+/// VPBlockBases.
+static bool hasDuplicates(const SmallVectorImpl<VPBlockBase *> &VPBlockVec) {
+ SmallDenseSet<const VPBlockBase *, 8> VPBlockSet;
+ for (const auto *Block : VPBlockVec) {
+ if (VPBlockSet.count(Block))
+ return true;
+ VPBlockSet.insert(Block);
+ }
+ return false;
+}
+
+/// Helper function that verifies the CFG invariants of the VPBlockBases within
+/// \p Region. Checks in this function are generic for VPBlockBases. They are
+/// not specific for VPBasicBlocks or VPRegionBlocks.
+static void verifyBlocksInRegion(const VPRegionBlock *Region) {
+ for (const VPBlockBase *VPB :
+ make_range(df_iterator<const VPBlockBase *>::begin(Region->getEntry()),
+ df_iterator<const VPBlockBase *>::end(Region->getExit()))) {
+ // Check block's parent.
+ assert(VPB->getParent() == Region && "VPBlockBase has wrong parent");
+
+ // Check block's successors.
+ const auto &Successors = VPB->getSuccessors();
+ // There must be only one instance of a successor in block's successor list.
+ // TODO: This won't work for switch statements.
+ assert(!hasDuplicates(Successors) &&
+ "Multiple instances of the same successor.");
+
+ for (const VPBlockBase *Succ : Successors) {
+ // There must be a bi-directional link between block and successor.
+ const auto &SuccPreds = Succ->getPredecessors();
+ assert(std::find(SuccPreds.begin(), SuccPreds.end(), VPB) !=
+ SuccPreds.end() &&
+ "Missing predecessor link.");
+ (void)SuccPreds;
+ }
+
+ // Check block's predecessors.
+ const auto &Predecessors = VPB->getPredecessors();
+ // There must be only one instance of a predecessor in block's predecessor
+ // list.
+ // TODO: This won't work for switch statements.
+ assert(!hasDuplicates(Predecessors) &&
+ "Multiple instances of the same predecessor.");
+
+ for (const VPBlockBase *Pred : Predecessors) {
+ // Block and predecessor must be inside the same region.
+ assert(Pred->getParent() == VPB->getParent() &&
+ "Predecessor is not in the same region.");
+
+ // There must be a bi-directional link between block and predecessor.
+ const auto &PredSuccs = Pred->getSuccessors();
+ assert(std::find(PredSuccs.begin(), PredSuccs.end(), VPB) !=
+ PredSuccs.end() &&
+ "Missing successor link.");
+ (void)PredSuccs;
+ }
+ }
+}
+
+/// Verify the CFG invariants of VPRegionBlock \p Region and its nested
+/// VPBlockBases. Do not recurse inside nested VPRegionBlocks.
+static void verifyRegion(const VPRegionBlock *Region) {
+ const VPBlockBase *Entry = Region->getEntry();
+ const VPBlockBase *Exit = Region->getExit();
+
+ // Entry and Exit shouldn't have any predecessor/successor, respectively.
+ assert(!Entry->getNumPredecessors() && "Region entry has predecessors.");
+ assert(!Exit->getNumSuccessors() && "Region exit has successors.");
+ (void)Entry;
+ (void)Exit;
+
+ verifyBlocksInRegion(Region);
+}
+
+/// Verify the CFG invariants of VPRegionBlock \p Region and its nested
+/// VPBlockBases. Recurse inside nested VPRegionBlocks.
+static void verifyRegionRec(const VPRegionBlock *Region) {
+ verifyRegion(Region);
+
+ // Recurse inside nested regions.
+ for (const VPBlockBase *VPB :
+ make_range(df_iterator<const VPBlockBase *>::begin(Region->getEntry()),
+ df_iterator<const VPBlockBase *>::end(Region->getExit()))) {
+ if (const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB))
+ verifyRegionRec(SubRegion);
+ }
+}
+
+void VPlanVerifier::verifyHierarchicalCFG(
+ const VPRegionBlock *TopRegion) const {
+ if (!EnableHCFGVerifier)
+ return;
+
+ DEBUG(dbgs() << "Verifying VPlan H-CFG.\n");
+ assert(!TopRegion->getParent() && "VPlan Top Region should have no parent.");
+ verifyRegionRec(TopRegion);
+}
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp
------------------------------------------------------------------------------
svn:keywords = Author Date Id Rev URL
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.cpp
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h?rev=332654&view=auto
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h (added)
+++ llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h Thu May 17 12:24:47 2018
@@ -0,0 +1,44 @@
+//===-- VPlanVerifier.h -----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file declares the class VPlanVerifier, which contains utility functions
+/// to check the consistency of a VPlan. This includes the following kinds of
+/// invariants:
+///
+/// 1. Region/Block invariants:
+/// - Region's entry/exit block must have no predecessors/successors,
+/// respectively.
+/// - Block's parent must be the region immediately containing the block.
+/// - Linked blocks must have a bi-directional link (successor/predecessor).
+/// - All predecessors/successors of a block must belong to the same region.
+/// - Blocks must have no duplicated successor/predecessor.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANVERIFIER_H
+#define LLVM_TRANSFORMS_VECTORIZE_VPLANVERIFIER_H
+
+#include "VPlan.h"
+
+namespace llvm {
+
+/// Class with utility functions that can be used to check the consistency and
+/// invariants of a VPlan, including the components of its H-CFG.
+class VPlanVerifier {
+public:
+ /// Verify the invariants of the H-CFG starting from \p TopRegion. The
+ /// verification process comprises the following steps:
+ /// 1. Region/Block verification: Check the Region/Block verification
+ /// invariants for every region in the H-CFG.
+ void verifyHierarchicalCFG(const VPRegionBlock *TopRegion) const;
+};
+} // namespace llvm
+
+#endif //LLVM_TRANSFORMS_VECTORIZE_VPLANVERIFIER_H
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h
------------------------------------------------------------------------------
svn:keywords = Author Date Id Rev URL
Propchange: llvm/trunk/lib/Transforms/Vectorize/VPlanVerifier.h
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: llvm/trunk/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll?rev=332654&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll (added)
+++ llvm/trunk/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll Thu May 17 12:24:47 2018
@@ -0,0 +1,51 @@
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -vplan-verify-hcfg -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s -check-prefix=VERIFIER
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s -check-prefix=NO-VERIFIER -allow-empty
+; REQUIRES: asserts
+
+; Verify that the stress testing flag for the VPlan H-CFG builder works as
+; expected with and without enabling the VPlan H-CFG Verifier.
+
+; VERIFIER: Verifying VPlan H-CFG.
+; NO-VERIFIER-NOT: Verifying VPlan H-CFG.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) {
+entry:
+ %cmp32 = icmp sgt i32 %N, 0
+ br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:
+ %cmp230 = icmp sgt i32 %M, 0
+ %0 = sext i32 %M to i64
+ %wide.trip.count = zext i32 %M to i64
+ %wide.trip.count38 = zext i32 %N to i64
+ br label %outer.body
+
+outer.body:
+ %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
+ br i1 %cmp230, label %inner.ph, label %outer.inc
+
+inner.ph:
+ %1 = mul nsw i64 %indvars.iv35, %0
+ br label %inner.body
+
+inner.body:
+ %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+ %2 = add nsw i64 %indvars.iv, %1
+ %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+ %3 = load i32, i32* %arrayidx, align 4
+ %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+ store i32 %3, i32* %arrayidx12, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:
+ %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+ %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
+ br i1 %exitcond39, label %for.end15, label %outer.body
+
+for.end15:
+ ret void
+}
More information about the llvm-commits
mailing list