[llvm] 0577612 - [VPlan] Use region for each loop in native path.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 1 02:41:26 PDT 2022
Author: Florian Hahn
Date: 2022-06-01T10:41:05+01:00
New Revision: 05776122b682684adac3de8bf736cc079989c55f
URL: https://github.com/llvm/llvm-project/commit/05776122b682684adac3de8bf736cc079989c55f
DIFF: https://github.com/llvm/llvm-project/commit/05776122b682684adac3de8bf736cc079989c55f.diff
LOG: [VPlan] Use region for each loop in native path.
This patch updates the VPlan native path to use VPRegionBlocks for all
loops in a loop nest. Up to now, only the outermost loop used a region.
This is a step towards unifying both paths and keep things consistent
between them. It also prepares various code-gen parts for modeling the
pre-header in the inner loop vectorizer (D121624).
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D123005
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d430129ff319c..7034b23998f89 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9089,31 +9089,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
[this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
DeadInstructions, *PSE.getSE());
- // Update plan to be compatible with the inner loop vectorizer for
- // code-generation.
- VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
- VPBasicBlock *Preheader = LoopRegion->getEntryBasicBlock();
- VPBasicBlock *Exiting = LoopRegion->getExitingBasicBlock();
- VPBlockBase *Latch = Exiting->getSinglePredecessor();
- VPBlockBase *Header = Preheader->getSingleSuccessor();
-
- // 1. Move preheader block out of main vector loop.
- Preheader->setParent(LoopRegion->getParent());
- VPBlockUtils::disconnectBlocks(Preheader, Header);
- VPBlockUtils::connectBlocks(Preheader, LoopRegion);
- Plan->setEntry(Preheader);
-
- // 2. Disconnect backedge and exiting block.
- VPBlockUtils::disconnectBlocks(Latch, Header);
- VPBlockUtils::disconnectBlocks(Latch, Exiting);
-
- // 3. Update entry and exiting of main vector loop region.
- LoopRegion->setEntry(Header);
- LoopRegion->setExiting(Latch);
-
- // 4. Remove exiting block.
- delete Exiting;
-
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DebugLoc(),
true, true);
return Plan;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index d2e844fb09525..f5cd8e5089206 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -265,19 +265,6 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
auto &PredVPSuccessors = PredVPBB->getSuccessors();
BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
- // In outer loop vectorization scenario, the predecessor BBlock may not yet
- // be visited(backedge). Mark the VPBasicBlock for fixup at the end of
- // vectorization. We do not encounter this case in inner loop vectorization
- // as we start out by building a loop skeleton with the vector loop header
- // and latch blocks. As a result, we never enter this function for the
- // header block in the non VPlan-native path.
- if (!PredBB) {
- assert(EnableVPlanNativePath &&
- "Unexpected null predecessor in non VPlan-native path");
- CFG.VPBBsToFix.push_back(PredVPBB);
- continue;
- }
-
assert(PredBB && "Predecessor basic-block not found building successor.");
auto *PredBBTerminator = PredBB->getTerminator();
LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
@@ -968,24 +955,6 @@ void VPlan::execute(VPTransformState *State) {
for (VPBlockBase *Block : depth_first(Entry))
Block->execute(State);
- // Setup branch terminator successors for VPBBs in VPBBsToFix based on
- // VPBB's successors.
- for (auto VPBB : State->CFG.VPBBsToFix) {
- assert(EnableVPlanNativePath &&
- "Unexpected VPBBsToFix in non VPlan-native path");
- BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
- assert(BB && "Unexpected null basic block for VPBB");
-
- unsigned Idx = 0;
- auto *BBTerminator = BB->getTerminator();
-
- for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
- VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
- BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
- ++Idx;
- }
- }
-
VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index eb3291ea0f311..9334e747141b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -316,10 +316,6 @@ struct VPTransformState {
/// of replication, maps the BasicBlock of the last replica created.
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
- /// Vector of VPBasicBlocks whose terminator instruction needs to be fixed
- /// up at the end of vector code generation.
- SmallVector<VPBasicBlock *, 8> VPBBsToFix;
-
CFGState() = default;
/// Returns the BasicBlock* mapped to the pre-header of the loop region
@@ -2667,13 +2663,9 @@ class VPlan {
/// Returns the VPRegionBlock of the vector loop.
VPRegionBlock *getVectorLoopRegion() {
- if (auto *R = dyn_cast<VPRegionBlock>(getEntry()))
- return R;
return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
}
const VPRegionBlock *getVectorLoopRegion() const {
- if (auto *R = dyn_cast<VPRegionBlock>(getEntry()))
- return R;
return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 7b4207c021df3..5ee97dfaf0f46 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -42,9 +42,6 @@ class PlainCFGBuilder {
// Vectorization plan that we are working on.
VPlan &Plan;
- // Output Top Region.
- VPRegionBlock *TopRegion = nullptr;
-
// Builder of the VPlan instruction-level representation.
VPBuilder VPIRBuilder;
@@ -59,6 +56,9 @@ class PlainCFGBuilder {
// Hold phi node's that need to be fixed once the plain CFG has been built.
SmallVector<PHINode *, 8> PhisToFix;
+ /// Maps loops in the original IR to their corresponding region.
+ DenseMap<Loop *, VPRegionBlock *> Loop2Region;
+
// Utility functions.
void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
void fixPhiNodes();
@@ -73,8 +73,9 @@ class PlainCFGBuilder {
PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
: TheLoop(Lp), LI(LI), Plan(P) {}
- // Build the plain CFG and return its Top Region.
- VPRegionBlock *buildPlainCFG();
+ /// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected
+ /// to a new VPRegionBlock (TopRegion) enclosing the plain CFG.
+ VPBasicBlock *buildPlainCFG();
};
} // anonymous namespace
@@ -106,19 +107,32 @@ void PlainCFGBuilder::fixPhiNodes() {
}
}
-// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
-// existing one if it was already created.
+// Create a new empty VPBasicBlock for an incoming BasicBlock in the region
+// corresponding to the containing loop or retrieve an existing one if it was
+// already created. If no region exists yet for the loop containing \p BB, a new
+// one is created.
VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
auto BlockIt = BB2VPBB.find(BB);
if (BlockIt != BB2VPBB.end())
// Retrieve existing VPBB.
return BlockIt->second;
+ // Get or create a region for the loop containing BB.
+ Loop *CurrentLoop = LI->getLoopFor(BB);
+ VPRegionBlock *ParentR = nullptr;
+ if (CurrentLoop) {
+ auto Iter = Loop2Region.insert({CurrentLoop, nullptr});
+ if (Iter.second)
+ Iter.first->second = new VPRegionBlock(
+ CurrentLoop->getHeader()->getName().str(), false /*isReplicator*/);
+ ParentR = Iter.first->second;
+ }
+
// Create new VPBB.
LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << BB->getName() << "\n");
VPBasicBlock *VPBB = new VPBasicBlock(BB->getName());
BB2VPBB[BB] = VPBB;
- VPBB->setParent(TopRegion);
+ VPBB->setParent(ParentR);
return VPBB;
}
@@ -237,11 +251,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
}
// Main interface to build the plain CFG.
-VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
- // 1. Create the Top Region. It will be the parent of all VPBBs.
- TopRegion = new VPRegionBlock("TopRegion", false /*isReplicator*/);
-
- // 2. Scan the body of the loop in a topological order to visit each basic
+VPBasicBlock *PlainCFGBuilder::buildPlainCFG() {
+ // 1. Scan the body of the loop in a topological order to visit each basic
// block after having visited its predecessor basic blocks. Create a VPBB for
// each BB and link it to its successor and predecessor VPBBs. Note that
// predecessors must be set in the same order as they are in the incomming IR.
@@ -250,11 +261,12 @@ VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
// Loop PH needs to be explicitly visited since it's not taken into account by
// LoopBlocksDFS.
- BasicBlock *PreheaderBB = TheLoop->getLoopPreheader();
- assert((PreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
+ BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
+ assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
"Unexpected loop preheader");
- VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(PreheaderBB);
- for (auto &I : *PreheaderBB) {
+ VPBasicBlock *ThePreheaderVPBB = getOrCreateVPBB(ThePreheaderBB);
+ ThePreheaderVPBB->setName("vector.ph");
+ for (auto &I : *ThePreheaderBB) {
if (I.getType()->isVoidTy())
continue;
IRDef2VPValue[&I] = Plan.getOrAddExternalDef(&I);
@@ -262,8 +274,7 @@ VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
// Create empty VPBB for Loop H so that we can link PH->H.
VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader());
HeaderVPBB->setName("vector.body");
- // Preheader's predecessors will be set during the loop RPO traversal below.
- PreheaderVPBB->setOneSuccessor(HeaderVPBB);
+ ThePreheaderVPBB->setOneSuccessor(HeaderVPBB);
LoopBlocksRPO RPO(TheLoop);
RPO.perform(LI);
@@ -310,30 +321,61 @@ VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
setVPBBPredsFromBB(VPBB, BB);
}
- // 3. Process outermost loop exit. We created an empty VPBB for the loop
+ // 2. Process outermost loop exit. We created an empty VPBB for the loop
// single exit BB during the RPO traversal of the loop body but Instructions
// weren't visited because it's not part of the the loop.
BasicBlock *LoopExitBB = TheLoop->getUniqueExitBlock();
assert(LoopExitBB && "Loops with multiple exits are not supported.");
VPBasicBlock *LoopExitVPBB = BB2VPBB[LoopExitBB];
- createVPInstructionsForVPBB(LoopExitVPBB, LoopExitBB);
// Loop exit was already set as successor of the loop exiting BB.
// We only set its predecessor VPBB now.
setVPBBPredsFromBB(LoopExitVPBB, LoopExitBB);
+ // 3. Fix up region blocks for loops. For each loop,
+ // * use the header block as entry to the corresponding region,
+ // * use the latch block as exit of the corresponding region,
+ // * set the region as successor of the loop pre-header, and
+ // * set the exit block as successor to the region.
+ SmallVector<Loop *> LoopWorkList;
+ LoopWorkList.push_back(TheLoop);
+ while (!LoopWorkList.empty()) {
+ Loop *L = LoopWorkList.pop_back_val();
+ BasicBlock *Header = L->getHeader();
+ BasicBlock *Exiting = L->getLoopLatch();
+ assert(Exiting == L->getExitingBlock() &&
+ "Latch must be the only exiting block");
+ VPRegionBlock *Region = Loop2Region[L];
+ VPBasicBlock *HeaderVPBB = getOrCreateVPBB(Header);
+ VPBasicBlock *ExitingVPBB = getOrCreateVPBB(Exiting);
+
+ // Disconnect backedge and pre-header from header.
+ VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(L->getLoopPreheader());
+ VPBlockUtils::disconnectBlocks(PreheaderVPBB, HeaderVPBB);
+ VPBlockUtils::disconnectBlocks(ExitingVPBB, HeaderVPBB);
+
+ Region->setParent(PreheaderVPBB->getParent());
+ Region->setEntry(HeaderVPBB);
+ VPBlockUtils::connectBlocks(PreheaderVPBB, Region);
+
+ // Disconnect exit block from exiting (=latch) block, set exiting block and
+ // connect region to exit block.
+ VPBasicBlock *ExitVPBB = getOrCreateVPBB(L->getExitBlock());
+ VPBlockUtils::disconnectBlocks(ExitingVPBB, ExitVPBB);
+ Region->setExiting(ExitingVPBB);
+ VPBlockUtils::connectBlocks(Region, ExitVPBB);
+
+ // Queue sub-loops for processing.
+ LoopWorkList.append(L->begin(), L->end());
+ }
// 4. The whole CFG has been built at this point so all the input Values must
// have a VPlan couterpart. Fix VPlan phi nodes by adding their corresponding
// VPlan operands.
fixPhiNodes();
- // 5. Final Top Region setup. Set outermost loop pre-header and single exit as
- // Top Region entry and exit.
- TopRegion->setEntry(PreheaderVPBB);
- TopRegion->setExiting(LoopExitVPBB);
- return TopRegion;
+ return ThePreheaderVPBB;
}
-VPRegionBlock *VPlanHCFGBuilder::buildPlainCFG() {
+VPBasicBlock *VPlanHCFGBuilder::buildPlainCFG() {
PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
return PCFGBuilder.buildPlainCFG();
}
@@ -341,10 +383,11 @@ VPRegionBlock *VPlanHCFGBuilder::buildPlainCFG() {
// Public interface to build a H-CFG.
void VPlanHCFGBuilder::buildHierarchicalCFG() {
// Build Top Region enclosing the plain CFG and set it as VPlan entry.
- VPRegionBlock *TopRegion = buildPlainCFG();
- Plan.setEntry(TopRegion);
+ VPBasicBlock *EntryVPBB = buildPlainCFG();
+ Plan.setEntry(EntryVPBB);
LLVM_DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan);
+ VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
Verifier.verifyHierarchicalCFG(TopRegion);
// Compute plain CFG dom tree for VPLInfo.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
index 23edf2e1dd8dd..2d52990af2688 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
@@ -57,9 +57,9 @@ class VPlanHCFGBuilder {
// are introduced.
VPDominatorTree VPDomTree;
- /// Build plain CFG for TheLoop. Return a new VPRegionBlock (TopRegion)
- /// enclosing the plain CFG.
- VPRegionBlock *buildPlainCFG();
+ /// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected
+ /// to a new VPRegionBlock (TopRegion) enclosing the plain CFG.
+ VPBasicBlock *buildPlainCFG();
public:
VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 85befe26e98a0..d4e0a5fae744e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -24,15 +24,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
GetIntOrFpInductionDescriptor,
SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
- auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
- ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
-
- for (VPBlockBase *Base : RPOT) {
- // Do not widen instructions in pre-header and exit blocks.
- if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
- continue;
-
- VPBasicBlock *VPBB = Base->getEntryBasicBlock();
+ ReversePostOrderTraversal<VPBlockRecursiveTraversalWrapper<VPBlockBase *>>
+ RPOT(Plan->getEntry());
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Introduce each ingredient into VPlan.
for (VPRecipeBase &Ingredient : llvm::make_early_inc_range(*VPBB)) {
VPValue *VPV = Ingredient.getVPSingleValue();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index bc6a9c973ba70..95aaf9bdad94f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -50,7 +50,7 @@ static void verifyBlocksInRegion(const VPRegionBlock *Region) {
assert(VPB->getParent() == Region && "VPBlockBase has wrong parent");
// Check block's condition bit.
- if (VPB->getNumSuccessors() > 1)
+ if (VPB->getNumSuccessors() > 1 || Region->getExitingBasicBlock() == VPB)
assert(VPB->getCondBit() && "Missing condition bit!");
else
assert(!VPB->getCondBit() && "Unexpected condition bit!");
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 038b15700cf1c..f3ca69bdedbd5 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -8,10 +8,10 @@
define void @foo(i64 %n) {
; CHECK: VPlan 'HCFGBuilder: Plain CFG
; CHECK-NEXT: {
-; CHECK-NEXT: <x1> TopRegion: {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: Successor(s): vector.body
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): outer.header
; CHECK-EMPTY:
+; CHECK-NEXT: <x1> outer.header: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2> ir<0> ir<%outer.iv>
@@ -19,25 +19,27 @@ define void @foo(i64 %n) {
; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv> ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
-; CHECK-NEXT: inner:
-; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
-; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr> ir<0> ir<%inner.iv> ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2>
-; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1>
-; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8>
-; CHECK-NEXT: Successor(s): outer.latch, inner
+; CHECK-NEXT: <x1> inner: {
+; CHECK-NEXT: inner:
+; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
+; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr> ir<0> ir<%inner.iv> ir<%outer.iv>
+; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2>
+; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1>
+; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8>
+; CHECK-NEXT: No successors
; CHECK-NEXT: CondBit: ir<%inner.ec> (inner)
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): outer.latch
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv> ir<1>
; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next> ir<8>
-; CHECK-NEXT: Successor(s): exit, vector.body
+; CHECK-NEXT: No successors
; CHECK-NEXT: CondBit: ir<%outer.ec> (outer.latch)
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): exit
; CHECK-EMPTY:
-; CHECK-NEXT: exit:
-; CHECK-NEXT: EMIT ret
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
+; CHECK-NEXT: exit:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index e7ca767e4b3a2..73510328cf1b9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -47,10 +47,14 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
EXPECT_EQ(1u, Entry->getNumSuccessors());
EXPECT_EQ(nullptr, Entry->getCondBit());
+ // Check that the region following the preheader is a single basic-block
+ // region (loop).
VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
EXPECT_EQ(7u, VecBB->size());
- EXPECT_EQ(2u, VecBB->getNumPredecessors());
- EXPECT_EQ(2u, VecBB->getNumSuccessors());
+ EXPECT_EQ(0u, VecBB->getNumPredecessors());
+ EXPECT_EQ(0u, VecBB->getNumSuccessors());
+ EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
+ EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
EXPECT_EQ(&*Plan, VecBB->getPlan());
auto Iter = VecBB->begin();
@@ -101,15 +105,15 @@ graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
node [shape=rect, fontname=Courier, fontsize=30]
edge [fontname=Courier, fontsize=30]
compound=true
- subgraph cluster_N0 {
+ N0 [label =
+ "vector.ph:\l" +
+ "Successor(s): for.body\l"
+ ]
+ N0 -> N1 [ label="" lhead=cluster_N2]
+ subgraph cluster_N2 {
fontname=Courier
- label="\<x1\> TopRegion"
+ label="\<x1\> for.body"
N1 [label =
- "entry:\l" +
- "Successor(s): vector.body\l"
- ]
- N1 -> N2 [ label=""]
- N2 [label =
"vector.body:\l" +
" WIDEN-PHI ir\<%indvars.iv\> = phi ir\<0\>, ir\<%indvars.iv.next\>\l" +
" EMIT ir\<%arr.idx\> = getelementptr ir\<%A\> ir\<%indvars.iv\>\l" +
@@ -118,17 +122,15 @@ compound=true
" EMIT store ir\<%res\> ir\<%arr.idx\>\l" +
" EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" +
" EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" +
- "Successor(s): vector.body, for.end\l" +
+ "No successors\l" +
"CondBit: ir\<%exitcond\> (vector.body)\l"
]
- N2 -> N2 [ label="T"]
- N2 -> N3 [ label="F"]
- N3 [label =
- "for.end:\l" +
- " EMIT ret\l" +
- "No successors\l"
- ]
}
+ N1 -> N3 [ label="" ltail=cluster_N2]
+ N3 [label =
+ "for.end:\l" +
+ "No successors\l"
+ ]
}
)";
EXPECT_EQ(ExpectedStr, FullDump);
@@ -174,10 +176,14 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
EXPECT_EQ(0u, Entry->getNumPredecessors());
EXPECT_EQ(1u, Entry->getNumSuccessors());
+ // Check that the region following the preheader is a single basic-block
+ // region (loop).
VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
EXPECT_EQ(7u, VecBB->size());
- EXPECT_EQ(2u, VecBB->getNumPredecessors());
- EXPECT_EQ(2u, VecBB->getNumSuccessors());
+ EXPECT_EQ(0u, VecBB->getNumPredecessors());
+ EXPECT_EQ(0u, VecBB->getNumSuccessors());
+ EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
+ EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
auto Iter = VecBB->begin();
EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index af0bef544e384..986f57a23d430 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -269,7 +269,7 @@ TEST_F(VPlanSlpTest, testSlpReuse_1) {
TEST_F(VPlanSlpTest, testSlpReuse_2) {
const char *ModuleString =
"%struct.Test = type { i32, i32 }\n"
- "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
+ "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
"nocapture readonly %B, %struct.Test* nocapture %C) {\n"
"entry:\n"
" br label %for.body\n"
@@ -290,11 +290,12 @@ TEST_F(VPlanSlpTest, testSlpReuse_2) {
" %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
"%indvars.iv, i32 1\n"
" store i32 %add1, i32* %C1, align 4\n"
+ " %use = add i32 %vA1, 1\n"
" %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
" %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
" br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
"for.cond.cleanup: ; preds = %for.body\n"
- " ret i32 %vA1\n"
+ " ret void\n"
"}\n";
Module &M = parseModule(ModuleString);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index c5b491627b582..31bd4418a174e 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -8,6 +8,7 @@
//===----------------------------------------------------------------------===//
#include "../lib/Transforms/Vectorize/VPlan.h"
+#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Instruction.h"
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 75fdab65fea43..b1139bd5bbdad 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -76,8 +76,8 @@ class VPlanTestBase : public testing::Test {
auto Plan = std::make_unique<VPlan>();
VPlanHCFGBuilder HCFGBuilder(LI->getLoopFor(LoopHeader), LI.get(), *Plan);
- VPRegionBlock *TopRegion = HCFGBuilder.buildPlainCFG();
- Plan->setEntry(TopRegion);
+ VPBasicBlock *EntryVPBB = HCFGBuilder.buildPlainCFG();
+ Plan->setEntry(EntryVPBB);
return Plan;
}
};
More information about the llvm-commits
mailing list