[llvm] e232d28 - [VPlan] Move plain CFG construction to VPlanConstruction. (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 18 13:52:28 PDT 2025
Author: Florian Hahn
Date: 2025-04-18T21:52:05+01:00
New Revision: e232d28eff876b1babbb731f1a0e5502c611a0ee
URL: https://github.com/llvm/llvm-project/commit/e232d28eff876b1babbb731f1a0e5502c611a0ee
DIFF: https://github.com/llvm/llvm-project/commit/e232d28eff876b1babbb731f1a0e5502c611a0ee.diff
LOG: [VPlan] Move plain CFG construction to VPlanConstruction. (NFC)
Follow-up as discussed in https://github.com/llvm/llvm-project/pull/129402.
After bc03d6cce257, the VPlanHCFGBuilder doesn't actually build a HCFG
any longer. Move what remains directly into VPlanConstruction.cpp.
Added:
Modified:
llvm/lib/Transforms/Vectorize/CMakeLists.txt
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
Removed:
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 7dac6d0059b26..0dc6a7d2f594f 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -24,7 +24,6 @@ add_llvm_component_library(LLVMVectorize
VPlan.cpp
VPlanAnalysis.cpp
VPlanConstruction.cpp
- VPlanHCFGBuilder.cpp
VPlanRecipes.cpp
VPlanSLP.cpp
VPlanTransforms.cpp
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bfceee46a237c..7a5f618d09e95 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -59,7 +59,6 @@
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
-#include "VPlanHCFGBuilder.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
@@ -9557,13 +9556,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
return !CM.requiresScalarEpilogue(VF.isVector());
},
Range);
- auto Plan = std::make_unique<VPlan>(OrigLoop);
- // Build hierarchical CFG.
- // TODO: Convert to VPlan-transform and consolidate all transforms for VPlan
- // creation.
- VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
- HCFGBuilder.buildPlainCFG();
-
+ DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
+ auto Plan = VPlanTransforms::buildPlainCFG(OrigLoop, *LI, VPB2IRBB);
VPlanTransforms::createLoopRegions(*Plan, Legal->getWidestInductionType(),
PSE, RequiresScalarEpilogueCheck,
CM.foldTailByMasking(), OrigLoop);
@@ -9642,7 +9636,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Handle VPBBs down to the latch.
if (VPBB == LoopRegion->getExiting()) {
- assert(!HCFGBuilder.getIRBBForVPB(VPBB) &&
+ assert(!VPB2IRBB.contains(VPBB) &&
"the latch block shouldn't have a corresponding IRBB");
VPBlockUtils::connectBlocks(PrevVPBB, VPBB);
break;
@@ -9658,7 +9652,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
// FIXME: At the moment, masks need to be placed at the beginning of the
// block, as blends introduced for phi nodes need to use it. The created
// blends should be sunk after the mask recipes.
- RecipeBuilder.createBlockInMask(HCFGBuilder.getIRBBForVPB(VPBB));
+ RecipeBuilder.createBlockInMask(VPB2IRBB.lookup(VPBB));
}
// Convert input VPInstructions to widened recipes.
@@ -9862,12 +9856,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
assert(!OrigLoop->isInnermost());
assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
- // Create new empty VPlan
- auto Plan = std::make_unique<VPlan>(OrigLoop);
- // Build hierarchical CFG
- VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
- HCFGBuilder.buildPlainCFG();
-
+ DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
+ auto Plan = VPlanTransforms::buildPlainCFG(OrigLoop, *LI, VPB2IRBB);
VPlanTransforms::createLoopRegions(*Plan, Legal->getWidestInductionType(),
PSE, true, false, OrigLoop);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1e687d0879f18..9fcccfcf8117f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -17,10 +17,355 @@
#include "VPlanDominatorTree.h"
#include "VPlanTransforms.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
+#define DEBUG_TYPE "vplan"
+
using namespace llvm;
+namespace {
+// Class that is used to build the plain CFG for the incoming IR.
+class PlainCFGBuilder {
+ // The outermost loop of the input loop nest considered for vectorization.
+ Loop *TheLoop;
+
+ // Loop Info analysis.
+ LoopInfo *LI;
+
+ // Vectorization plan that we are working on.
+ std::unique_ptr<VPlan> Plan;
+
+ // Builder of the VPlan instruction-level representation.
+ VPBuilder VPIRBuilder;
+
+ // NOTE: The following maps are intentionally destroyed after the plain CFG
+ // construction because subsequent VPlan-to-VPlan transformation may
+ // invalidate them.
+ // Map incoming BasicBlocks to their newly-created VPBasicBlocks.
+ DenseMap<BasicBlock *, VPBasicBlock *> BB2VPBB;
+ // Map incoming Value definitions to their newly-created VPValues.
+ DenseMap<Value *, VPValue *> IRDef2VPValue;
+
+ // Hold phi node's that need to be fixed once the plain CFG has been built.
+ SmallVector<PHINode *, 8> PhisToFix;
+
+ // Utility functions.
+ void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
+ void fixHeaderPhis();
+ VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
+#ifndef NDEBUG
+ bool isExternalDef(Value *Val);
+#endif
+ VPValue *getOrCreateVPOperand(Value *IRVal);
+ void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
+
+public:
+ PlainCFGBuilder(Loop *Lp, LoopInfo *LI)
+ : TheLoop(Lp), LI(LI), Plan(std::make_unique<VPlan>(Lp)) {}
+
+ /// Build plain CFG for TheLoop and connects it to Plan's entry.
+ std::unique_ptr<VPlan>
+ buildPlainCFG(DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);
+};
+} // anonymous namespace
+
+// Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB
+// must have no predecessors.
+void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
+ // Collect VPBB predecessors.
+ SmallVector<VPBlockBase *, 2> VPBBPreds;
+ for (BasicBlock *Pred : predecessors(BB))
+ VPBBPreds.push_back(getOrCreateVPBB(Pred));
+ VPBB->setPredecessors(VPBBPreds);
+}
+
+static bool isHeaderBB(BasicBlock *BB, Loop *L) {
+ return L && BB == L->getHeader();
+}
+
+// Add operands to VPInstructions representing phi nodes from the input IR.
+void PlainCFGBuilder::fixHeaderPhis() {
+ for (auto *Phi : PhisToFix) {
+ assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
+ VPValue *VPVal = IRDef2VPValue[Phi];
+ assert(isa<VPWidenPHIRecipe>(VPVal) &&
+ "Expected WidenPHIRecipe for phi node.");
+ auto *VPPhi = cast<VPWidenPHIRecipe>(VPVal);
+ assert(VPPhi->getNumOperands() == 0 &&
+ "Expected VPInstruction with no operands.");
+ assert(isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent())) &&
+ "Expected Phi in header block.");
+ assert(Phi->getNumOperands() == 2 &&
+ "header phi must have exactly 2 operands");
+ for (BasicBlock *Pred : predecessors(Phi->getParent()))
+ VPPhi->addOperand(
+ getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
+ }
+}
+
+// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
+// existing one if it was already created.
+VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
+ if (auto *VPBB = BB2VPBB.lookup(BB)) {
+ // Retrieve existing VPBB.
+ return VPBB;
+ }
+
+ // Create new VPBB.
+ StringRef Name = BB->getName();
+ LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << Name << "\n");
+ VPBasicBlock *VPBB = Plan->createVPBasicBlock(Name);
+ BB2VPBB[BB] = VPBB;
+ return VPBB;
+}
+
+#ifndef NDEBUG
+// Return true if \p Val is considered an external definition. An external
+// definition is either:
+// 1. A Value that is not an Instruction. This will be refined in the future.
+// 2. An Instruction that is outside of the CFG snippet represented in VPlan,
+// i.e., is not part of: a) the loop nest, b) outermost loop PH and, c)
+// outermost loop exits.
+bool PlainCFGBuilder::isExternalDef(Value *Val) {
+ // All the Values that are not Instructions are considered external
+ // definitions for now.
+ Instruction *Inst = dyn_cast<Instruction>(Val);
+ if (!Inst)
+ return true;
+
+ BasicBlock *InstParent = Inst->getParent();
+ assert(InstParent && "Expected instruction parent.");
+
+ // Check whether Instruction definition is in loop PH.
+ BasicBlock *PH = TheLoop->getLoopPreheader();
+ assert(PH && "Expected loop pre-header.");
+
+ if (InstParent == PH)
+ // Instruction definition is in outermost loop PH.
+ return false;
+
+ // Check whether Instruction definition is in a loop exit.
+ SmallVector<BasicBlock *> ExitBlocks;
+ TheLoop->getExitBlocks(ExitBlocks);
+ if (is_contained(ExitBlocks, InstParent)) {
+ // Instruction definition is in outermost loop exit.
+ return false;
+ }
+
+ // Check whether Instruction definition is in loop body.
+ return !TheLoop->contains(Inst);
+}
+#endif
+
+// Create a new VPValue or retrieve an existing one for the Instruction's
+// operand \p IRVal. This function must only be used to create/retrieve VPValues
+// for *Instruction's operands* and not to create regular VPInstruction's. For
+// the latter, please, look at 'createVPInstructionsForVPBB'.
+VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
+ auto VPValIt = IRDef2VPValue.find(IRVal);
+ if (VPValIt != IRDef2VPValue.end())
+ // Operand has an associated VPInstruction or VPValue that was previously
+ // created.
+ return VPValIt->second;
+
+ // Operand doesn't have a previously created VPInstruction/VPValue. This
+ // means that operand is:
+ // A) a definition external to VPlan,
+ // B) any other Value without specific representation in VPlan.
+ // For now, we use VPValue to represent A and B and classify both as external
+ // definitions. We may introduce specific VPValue subclasses for them in the
+ // future.
+ assert(isExternalDef(IRVal) && "Expected external definition as operand.");
+
+ // A and B: Create VPValue and add it to the pool of external definitions and
+ // to the Value->VPValue map.
+ VPValue *NewVPVal = Plan->getOrAddLiveIn(IRVal);
+ IRDef2VPValue[IRVal] = NewVPVal;
+ return NewVPVal;
+}
+
+// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
+// counterpart. This function must be invoked in RPO so that the operands of a
+// VPInstruction in \p BB have been visited before (except for Phi nodes).
+void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
+ BasicBlock *BB) {
+ VPIRBuilder.setInsertPoint(VPBB);
+ // TODO: Model and preserve debug intrinsics in VPlan.
+ for (Instruction &InstRef : BB->instructionsWithoutDebug(false)) {
+ Instruction *Inst = &InstRef;
+
+ // There shouldn't be any VPValue for Inst at this point. Otherwise, we
+ // visited Inst when we shouldn't, breaking the RPO traversal order.
+ assert(!IRDef2VPValue.count(Inst) &&
+ "Instruction shouldn't have been visited.");
+
+ if (auto *Br = dyn_cast<BranchInst>(Inst)) {
+ if (TheLoop->getLoopLatch() == BB ||
+ any_of(successors(BB),
+ [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
+ continue;
+
+ // Conditional branch instruction are represented using BranchOnCond
+ // recipes.
+ if (Br->isConditional()) {
+ VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
+ VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst);
+ }
+
+ // Skip the rest of the Instruction processing for Branch instructions.
+ continue;
+ }
+
+ if (auto *SI = dyn_cast<SwitchInst>(Inst)) {
+ SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
+ for (auto Case : SI->cases())
+ Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
+ VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst);
+ continue;
+ }
+
+ VPSingleDefRecipe *NewR;
+ if (auto *Phi = dyn_cast<PHINode>(Inst)) {
+ // Phi node's operands may have not been visited at this point. We create
+ // an empty VPInstruction that we will fix once the whole plain CFG has
+ // been built.
+ NewR = new VPWidenPHIRecipe(Phi, nullptr, Phi->getDebugLoc(), "vec.phi");
+ VPBB->appendRecipe(NewR);
+ if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
+ // Header phis need to be fixed after the VPBB for the latch has been
+ // created.
+ PhisToFix.push_back(Phi);
+ } else {
+ // Add operands for VPPhi in the order matching its predecessors in
+ // VPlan.
+ DenseMap<const VPBasicBlock *, VPValue *> VPPredToIncomingValue;
+ for (unsigned I = 0; I != Phi->getNumOperands(); ++I) {
+ VPPredToIncomingValue[BB2VPBB[Phi->getIncomingBlock(I)]] =
+ getOrCreateVPOperand(Phi->getIncomingValue(I));
+ }
+ for (VPBlockBase *Pred : VPBB->getPredecessors())
+ NewR->addOperand(
+ VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock()));
+ }
+ } else {
+ // Translate LLVM-IR operands into VPValue operands and set them in the
+ // new VPInstruction.
+ SmallVector<VPValue *, 4> VPOperands;
+ for (Value *Op : Inst->operands())
+ VPOperands.push_back(getOrCreateVPOperand(Op));
+
+ // Build VPInstruction for any arbitrary Instruction without specific
+ // representation in VPlan.
+ NewR = cast<VPInstruction>(
+ VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
+ }
+
+ IRDef2VPValue[Inst] = NewR;
+ }
+}
+
+// Main interface to build the plain CFG.
+std::unique_ptr<VPlan> PlainCFGBuilder::buildPlainCFG(
+ DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
+ VPIRBasicBlock *Entry = cast<VPIRBasicBlock>(Plan->getEntry());
+ BB2VPBB[Entry->getIRBasicBlock()] = Entry;
+
+ // 1. Scan the body of the loop in a topological order to visit each basic
+ // block after having visited its predecessor basic blocks. Create a VPBB for
+ // each BB and link it to its successor and predecessor VPBBs. Note that
+ // predecessors must be set in the same order as they are in the incomming IR.
+ // Otherwise, there might be problems with existing phi nodes and algorithm
+ // based on predecessors traversal.
+
+ // Loop PH needs to be explicitly visited since it's not taken into account by
+ // LoopBlocksDFS.
+ BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
+ assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
+ "Unexpected loop preheader");
+ for (auto &I : *ThePreheaderBB) {
+ if (I.getType()->isVoidTy())
+ continue;
+ IRDef2VPValue[&I] = Plan->getOrAddLiveIn(&I);
+ }
+
+ LoopBlocksRPO RPO(TheLoop);
+ RPO.perform(LI);
+
+ for (BasicBlock *BB : RPO) {
+ // Create or retrieve the VPBasicBlock for this BB.
+ VPBasicBlock *VPBB = getOrCreateVPBB(BB);
+ Loop *LoopForBB = LI->getLoopFor(BB);
+ // Set VPBB predecessors in the same order as they are in the incoming BB.
+ setVPBBPredsFromBB(VPBB, BB);
+
+ // Create VPInstructions for BB.
+ createVPInstructionsForVPBB(VPBB, BB);
+
+ // Set VPBB successors. We create empty VPBBs for successors if they don't
+ // exist already. Recipes will be created when the successor is visited
+ // during the RPO traversal.
+ if (auto *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
+ SmallVector<VPBlockBase *> Succs = {
+ getOrCreateVPBB(SI->getDefaultDest())};
+ for (auto Case : SI->cases())
+ Succs.push_back(getOrCreateVPBB(Case.getCaseSuccessor()));
+ VPBB->setSuccessors(Succs);
+ continue;
+ }
+ auto *BI = cast<BranchInst>(BB->getTerminator());
+ unsigned NumSuccs = succ_size(BB);
+ if (NumSuccs == 1) {
+ VPBB->setOneSuccessor(getOrCreateVPBB(BB->getSingleSuccessor()));
+ continue;
+ }
+ assert(BI->isConditional() && NumSuccs == 2 && BI->isConditional() &&
+ "block must have conditional branch with 2 successors");
+
+ BasicBlock *IRSucc0 = BI->getSuccessor(0);
+ BasicBlock *IRSucc1 = BI->getSuccessor(1);
+ VPBasicBlock *Successor0 = getOrCreateVPBB(IRSucc0);
+ VPBasicBlock *Successor1 = getOrCreateVPBB(IRSucc1);
+
+ // Don't connect any blocks outside the current loop except the latches for
+ // inner loops.
+ // TODO: Also connect exit blocks during initial VPlan construction.
+ if (LoopForBB == TheLoop || BB != LoopForBB->getLoopLatch()) {
+ if (!LoopForBB->contains(IRSucc0)) {
+ VPBB->setOneSuccessor(Successor1);
+ continue;
+ }
+ if (!LoopForBB->contains(IRSucc1)) {
+ VPBB->setOneSuccessor(Successor0);
+ continue;
+ }
+ }
+
+ VPBB->setTwoSuccessors(Successor0, Successor1);
+ }
+
+ // 2. The whole CFG has been built at this point so all the input Values must
+ // have a VPlan counterpart. Fix VPlan header phi by adding their
+ // corresponding VPlan operands.
+ fixHeaderPhis();
+
+ Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
+ Plan->getEntry()->setPlan(&*Plan);
+
+ for (const auto &[IRBB, VPB] : BB2VPBB)
+ VPB2IRBB[VPB] = IRBB;
+
+ LLVM_DEBUG(Plan->setName("Plain CFG\n"); dbgs() << *Plan);
+ return std::move(Plan);
+}
+
+std::unique_ptr<VPlan> VPlanTransforms::buildPlainCFG(
+ Loop *TheLoop, LoopInfo &LI,
+ DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
+ PlainCFGBuilder Builder(TheLoop, &LI);
+ return Builder.buildPlainCFG(VPB2IRBB);
+}
+
/// Checks if \p HeaderVPB is a loop header block in the plain CFG; that is, it
/// has exactly 2 predecessors (preheader and latch), where the block
/// dominates the latch and the preheader dominates the block. If it is a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
deleted file mode 100644
index 5bacd2d4e6d88..0000000000000
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ /dev/null
@@ -1,368 +0,0 @@
-//===-- VPlanHCFGBuilder.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This file implements the construction of a VPlan-based Hierarchical CFG
-/// (H-CFG) for an incoming IR. This construction comprises the following
-/// components and steps:
-//
-/// 1. PlainCFGBuilder class: builds a plain VPBasicBlock-based CFG that
-/// faithfully represents the CFG in the incoming IR.
-/// NOTE: At this point, there is a direct correspondence between all the
-/// VPBasicBlocks created for the initial plain CFG and the incoming
-/// BasicBlocks. However, this might change in the future.
-///
-//===----------------------------------------------------------------------===//
-
-#include "VPlanHCFGBuilder.h"
-#include "LoopVectorizationPlanner.h"
-#include "VPlanCFG.h"
-#include "llvm/Analysis/LoopIterator.h"
-
-#define DEBUG_TYPE "loop-vectorize"
-
-using namespace llvm;
-
-namespace {
-// Class that is used to build the plain CFG for the incoming IR.
-class PlainCFGBuilder {
-private:
- // The outermost loop of the input loop nest considered for vectorization.
- Loop *TheLoop;
-
- // Loop Info analysis.
- LoopInfo *LI;
-
- // Vectorization plan that we are working on.
- VPlan &Plan;
-
- // Builder of the VPlan instruction-level representation.
- VPBuilder VPIRBuilder;
-
- // NOTE: The following maps are intentionally destroyed after the plain CFG
- // construction because subsequent VPlan-to-VPlan transformation may
- // invalidate them.
- // Map incoming BasicBlocks to their newly-created VPBasicBlocks.
- DenseMap<BasicBlock *, VPBasicBlock *> BB2VPBB;
- // Map incoming Value definitions to their newly-created VPValues.
- DenseMap<Value *, VPValue *> IRDef2VPValue;
-
- // Hold phi node's that need to be fixed once the plain CFG has been built.
- SmallVector<PHINode *, 8> PhisToFix;
-
- // Utility functions.
- void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
- void fixHeaderPhis();
- VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
-#ifndef NDEBUG
- bool isExternalDef(Value *Val);
-#endif
- VPValue *getOrCreateVPOperand(Value *IRVal);
- void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
-
-public:
- PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
- : TheLoop(Lp), LI(LI), Plan(P) {}
-
- /// Build plain CFG for TheLoop and connects it to Plan's entry.
- void buildPlainCFG(DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);
-};
-} // anonymous namespace
-
-// Set predecessors of \p VPBB in the same order as they are in \p BB. \p VPBB
-// must have no predecessors.
-void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) {
- // Collect VPBB predecessors.
- SmallVector<VPBlockBase *, 2> VPBBPreds;
- for (BasicBlock *Pred : predecessors(BB))
- VPBBPreds.push_back(getOrCreateVPBB(Pred));
- VPBB->setPredecessors(VPBBPreds);
-}
-
-static bool isHeaderBB(BasicBlock *BB, Loop *L) {
- return L && BB == L->getHeader();
-}
-
-// Add operands to VPInstructions representing phi nodes from the input IR.
-void PlainCFGBuilder::fixHeaderPhis() {
- for (auto *Phi : PhisToFix) {
- assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
- VPValue *VPVal = IRDef2VPValue[Phi];
- assert(isa<VPWidenPHIRecipe>(VPVal) &&
- "Expected WidenPHIRecipe for phi node.");
- auto *VPPhi = cast<VPWidenPHIRecipe>(VPVal);
- assert(VPPhi->getNumOperands() == 0 &&
- "Expected VPInstruction with no operands.");
- assert(isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent())) &&
- "Expected Phi in header block.");
- assert(Phi->getNumOperands() == 2 &&
- "header phi must have exactly 2 operands");
- for (BasicBlock *Pred : predecessors(Phi->getParent()))
- VPPhi->addOperand(
- getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
- }
-}
-
-// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
-// existing one if it was already created.
-VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
- if (auto *VPBB = BB2VPBB.lookup(BB)) {
- // Retrieve existing VPBB.
- return VPBB;
- }
-
- // Create new VPBB.
- StringRef Name = BB->getName();
- LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << Name << "\n");
- VPBasicBlock *VPBB = Plan.createVPBasicBlock(Name);
- BB2VPBB[BB] = VPBB;
- return VPBB;
-}
-
-#ifndef NDEBUG
-// Return true if \p Val is considered an external definition. An external
-// definition is either:
-// 1. A Value that is not an Instruction. This will be refined in the future.
-// 2. An Instruction that is outside of the CFG snippet represented in VPlan,
-// i.e., is not part of: a) the loop nest, b) outermost loop PH and, c)
-// outermost loop exits.
-bool PlainCFGBuilder::isExternalDef(Value *Val) {
- // All the Values that are not Instructions are considered external
- // definitions for now.
- Instruction *Inst = dyn_cast<Instruction>(Val);
- if (!Inst)
- return true;
-
- BasicBlock *InstParent = Inst->getParent();
- assert(InstParent && "Expected instruction parent.");
-
- // Check whether Instruction definition is in loop PH.
- BasicBlock *PH = TheLoop->getLoopPreheader();
- assert(PH && "Expected loop pre-header.");
-
- if (InstParent == PH)
- // Instruction definition is in outermost loop PH.
- return false;
-
- // Check whether Instruction definition is in a loop exit.
- SmallVector<BasicBlock *> ExitBlocks;
- TheLoop->getExitBlocks(ExitBlocks);
- if (is_contained(ExitBlocks, InstParent)) {
- // Instruction definition is in outermost loop exit.
- return false;
- }
-
- // Check whether Instruction definition is in loop body.
- return !TheLoop->contains(Inst);
-}
-#endif
-
-// Create a new VPValue or retrieve an existing one for the Instruction's
-// operand \p IRVal. This function must only be used to create/retrieve VPValues
-// for *Instruction's operands* and not to create regular VPInstruction's. For
-// the latter, please, look at 'createVPInstructionsForVPBB'.
-VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
- auto VPValIt = IRDef2VPValue.find(IRVal);
- if (VPValIt != IRDef2VPValue.end())
- // Operand has an associated VPInstruction or VPValue that was previously
- // created.
- return VPValIt->second;
-
- // Operand doesn't have a previously created VPInstruction/VPValue. This
- // means that operand is:
- // A) a definition external to VPlan,
- // B) any other Value without specific representation in VPlan.
- // For now, we use VPValue to represent A and B and classify both as external
- // definitions. We may introduce specific VPValue subclasses for them in the
- // future.
- assert(isExternalDef(IRVal) && "Expected external definition as operand.");
-
- // A and B: Create VPValue and add it to the pool of external definitions and
- // to the Value->VPValue map.
- VPValue *NewVPVal = Plan.getOrAddLiveIn(IRVal);
- IRDef2VPValue[IRVal] = NewVPVal;
- return NewVPVal;
-}
-
-// Create new VPInstructions in a VPBasicBlock, given its BasicBlock
-// counterpart. This function must be invoked in RPO so that the operands of a
-// VPInstruction in \p BB have been visited before (except for Phi nodes).
-void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
- BasicBlock *BB) {
- VPIRBuilder.setInsertPoint(VPBB);
- // TODO: Model and preserve debug intrinsics in VPlan.
- for (Instruction &InstRef : BB->instructionsWithoutDebug(false)) {
- Instruction *Inst = &InstRef;
-
- // There shouldn't be any VPValue for Inst at this point. Otherwise, we
- // visited Inst when we shouldn't, breaking the RPO traversal order.
- assert(!IRDef2VPValue.count(Inst) &&
- "Instruction shouldn't have been visited.");
-
- if (auto *Br = dyn_cast<BranchInst>(Inst)) {
- if (TheLoop->getLoopLatch() == BB ||
- any_of(successors(BB),
- [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
- continue;
-
- // Conditional branch instruction are represented using BranchOnCond
- // recipes.
- if (Br->isConditional()) {
- VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
- VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst);
- }
-
- // Skip the rest of the Instruction processing for Branch instructions.
- continue;
- }
-
- if (auto *SI = dyn_cast<SwitchInst>(Inst)) {
- SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
- for (auto Case : SI->cases())
- Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
- VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst);
- continue;
- }
-
- VPSingleDefRecipe *NewR;
- if (auto *Phi = dyn_cast<PHINode>(Inst)) {
- // Phi node's operands may have not been visited at this point. We create
- // an empty VPInstruction that we will fix once the whole plain CFG has
- // been built.
- NewR = new VPWidenPHIRecipe(Phi, nullptr, Phi->getDebugLoc(), "vec.phi");
- VPBB->appendRecipe(NewR);
- if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
- // Header phis need to be fixed after the VPBB for the latch has been
- // created.
- PhisToFix.push_back(Phi);
- } else {
- // Add operands for VPPhi in the order matching its predecessors in
- // VPlan.
- DenseMap<const VPBasicBlock *, VPValue *> VPPredToIncomingValue;
- for (unsigned I = 0; I != Phi->getNumOperands(); ++I) {
- VPPredToIncomingValue[BB2VPBB[Phi->getIncomingBlock(I)]] =
- getOrCreateVPOperand(Phi->getIncomingValue(I));
- }
- for (VPBlockBase *Pred : VPBB->getPredecessors())
- NewR->addOperand(
- VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock()));
- }
- } else {
- // Translate LLVM-IR operands into VPValue operands and set them in the
- // new VPInstruction.
- SmallVector<VPValue *, 4> VPOperands;
- for (Value *Op : Inst->operands())
- VPOperands.push_back(getOrCreateVPOperand(Op));
-
- // Build VPInstruction for any arbitrary Instruction without specific
- // representation in VPlan.
- NewR = cast<VPInstruction>(
- VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst));
- }
-
- IRDef2VPValue[Inst] = NewR;
- }
-}
-
-// Main interface to build the plain CFG.
-void PlainCFGBuilder::buildPlainCFG(
- DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
- VPIRBasicBlock *Entry = cast<VPIRBasicBlock>(Plan.getEntry());
- BB2VPBB[Entry->getIRBasicBlock()] = Entry;
-
- // 1. Scan the body of the loop in a topological order to visit each basic
- // block after having visited its predecessor basic blocks. Create a VPBB for
- // each BB and link it to its successor and predecessor VPBBs. Note that
- // predecessors must be set in the same order as they are in the incomming IR.
- // Otherwise, there might be problems with existing phi nodes and algorithm
- // based on predecessors traversal.
-
- // Loop PH needs to be explicitly visited since it's not taken into account by
- // LoopBlocksDFS.
- BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
- assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
- "Unexpected loop preheader");
- for (auto &I : *ThePreheaderBB) {
- if (I.getType()->isVoidTy())
- continue;
- IRDef2VPValue[&I] = Plan.getOrAddLiveIn(&I);
- }
-
- LoopBlocksRPO RPO(TheLoop);
- RPO.perform(LI);
-
- for (BasicBlock *BB : RPO) {
- // Create or retrieve the VPBasicBlock for this BB.
- VPBasicBlock *VPBB = getOrCreateVPBB(BB);
- Loop *LoopForBB = LI->getLoopFor(BB);
- // Set VPBB predecessors in the same order as they are in the incoming BB.
- setVPBBPredsFromBB(VPBB, BB);
-
- // Create VPInstructions for BB.
- createVPInstructionsForVPBB(VPBB, BB);
-
- // Set VPBB successors. We create empty VPBBs for successors if they don't
- // exist already. Recipes will be created when the successor is visited
- // during the RPO traversal.
- if (auto *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
- SmallVector<VPBlockBase *> Succs = {
- getOrCreateVPBB(SI->getDefaultDest())};
- for (auto Case : SI->cases())
- Succs.push_back(getOrCreateVPBB(Case.getCaseSuccessor()));
- VPBB->setSuccessors(Succs);
- continue;
- }
- auto *BI = cast<BranchInst>(BB->getTerminator());
- unsigned NumSuccs = succ_size(BB);
- if (NumSuccs == 1) {
- VPBB->setOneSuccessor(getOrCreateVPBB(BB->getSingleSuccessor()));
- continue;
- }
- assert(BI->isConditional() && NumSuccs == 2 && BI->isConditional() &&
- "block must have conditional branch with 2 successors");
-
- BasicBlock *IRSucc0 = BI->getSuccessor(0);
- BasicBlock *IRSucc1 = BI->getSuccessor(1);
- VPBasicBlock *Successor0 = getOrCreateVPBB(IRSucc0);
- VPBasicBlock *Successor1 = getOrCreateVPBB(IRSucc1);
-
- // Don't connect any blocks outside the current loop except the latches for
- // inner loops.
- // TODO: Also connect exit blocks during initial VPlan construction.
- if (LoopForBB == TheLoop || BB != LoopForBB->getLoopLatch()) {
- if (!LoopForBB->contains(IRSucc0)) {
- VPBB->setOneSuccessor(Successor1);
- continue;
- }
- if (!LoopForBB->contains(IRSucc1)) {
- VPBB->setOneSuccessor(Successor0);
- continue;
- }
- }
-
- VPBB->setTwoSuccessors(Successor0, Successor1);
- }
-
- // 2. The whole CFG has been built at this point so all the input Values must
- // have a VPlan counterpart. Fix VPlan header phi by adding their
- // corresponding VPlan operands.
- fixHeaderPhis();
-
- Plan.getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader()));
- Plan.getEntry()->setPlan(&Plan);
-
- for (const auto &[IRBB, VPB] : BB2VPBB)
- VPB2IRBB[VPB] = IRBB;
-
- LLVM_DEBUG(Plan.setName("Plain CFG\n"); dbgs() << Plan);
-}
-
-void VPlanHCFGBuilder::buildPlainCFG() {
- PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
- PCFGBuilder.buildPlainCFG(VPB2IRBB);
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
deleted file mode 100644
index f2e90d3f4d9b3..0000000000000
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
+++ /dev/null
@@ -1,73 +0,0 @@
-//===-- VPlanHCFGBuilder.h --------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This file defines the VPlanHCFGBuilder class which contains the public
-/// interface (buildHierarchicalCFG) to build a VPlan-based Hierarchical CFG
-/// (H-CFG) for an incoming IR.
-///
-/// A H-CFG in VPlan is a control-flow graph whose nodes are VPBasicBlocks
-/// and/or VPRegionBlocks (i.e., other H-CFGs). The outermost H-CFG of a VPlan
-/// consists of a VPRegionBlock, denoted Top Region, which encloses any other
-/// VPBlockBase in the H-CFG. This guarantees that any VPBlockBase in the H-CFG
-/// other than the Top Region will have a parent VPRegionBlock and allows us
-/// to easily add more nodes before/after the main vector loop (such as the
-/// reduction epilogue).
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
-#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
-
-#include "llvm/ADT/DenseMap.h"
-
-namespace llvm {
-
-class Loop;
-class LoopInfo;
-class VPlan;
-class VPlanTestIRBase;
-class VPBlockBase;
-class BasicBlock;
-
-/// Main class to build the VPlan H-CFG for an incoming IR.
-class VPlanHCFGBuilder {
- friend VPlanTestIRBase;
-
-private:
- // The outermost loop of the input loop nest considered for vectorization.
- Loop *TheLoop;
-
- // Loop Info analysis.
- LoopInfo *LI;
-
- // The VPlan that will contain the H-CFG we are building.
- VPlan &Plan;
-
- /// Map of create VP blocks to their input IR basic blocks, if they have been
- /// created for a input IR basic block.
- DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
-
-public:
- VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
- : TheLoop(Lp), LI(LI), Plan(P) {}
-
- /// Build plain CFG for TheLoop and connects it to Plan's entry.
- void buildPlainCFG();
-
- /// Return the input IR BasicBlock corresponding to \p VPB. Returns nullptr if
- /// there is no such corresponding block.
- /// FIXME: This is a temporary workaround to drive the createBlockInMask.
- /// Remove once mask creation is done on VPlan.
- BasicBlock *getIRBBForVPB(const VPBlockBase *VPB) const {
- return VPB2IRBB.lookup(VPB);
- }
-};
-} // namespace llvm
-
-#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_VPLANHCFGBUILDER_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 65f3c837f7721..80c1159a9d5da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -53,6 +53,10 @@ struct VPlanTransforms {
verifyVPlanIsValid(Plan);
}
+ static std::unique_ptr<VPlan>
+ buildPlainCFG(Loop *TheLoop, LoopInfo &LI,
+ DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);
+
/// Replace loops in \p Plan's flat CFG with VPRegionBlocks, turing \p Plan's
/// flat CFG into a hierarchical CFG. It also creates a VPValue expression for
/// the original trip count. It will also introduce a dedicated VPBasicBlock
diff --git a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
index 29aeb7c4e97f9..de536bf39c2ca 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=vplan -disable-output 2>&1 | FileCheck %s
; REQUIRES: asserts
; Verify that the stress testing flag for the VPlan H-CFG builder works as
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index cd8bd4a3565e4..1ffd1a6a7a9b9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -8,7 +8,6 @@
#include "../lib/Transforms/Vectorize/VPlanSLP.h"
#include "../lib/Transforms/Vectorize/VPlan.h"
-#include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
#include "VPlanTestBase.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 486296535996b..d49483f0ebf88 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -13,7 +13,6 @@
#define LLVM_UNITTESTS_TRANSFORMS_VECTORIZE_VPLANTESTBASE_H
#include "../lib/Transforms/Vectorize/VPlan.h"
-#include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
#include "../lib/Transforms/Vectorize/VPlanTransforms.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
@@ -71,9 +70,8 @@ class VPlanTestIRBase : public testing::Test {
Loop *L = LI->getLoopFor(LoopHeader);
PredicatedScalarEvolution PSE(*SE, *L);
- auto Plan = std::make_unique<VPlan>(L);
- VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan);
- HCFGBuilder.buildPlainCFG();
+ DenseMap<VPBlockBase *, BasicBlock *> VPB2IRBB;
+ auto Plan = VPlanTransforms::buildPlainCFG(L, *LI, VPB2IRBB);
VPlanTransforms::createLoopRegions(*Plan, IntegerType::get(*Ctx, 64), PSE,
true, false, L);
return Plan;
More information about the llvm-commits
mailing list