[llvm] [VPlan] Construct immutable VPIRBBs for exit blocks at construction(NFC) (PR #128374)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 25 03:48:20 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/128374
>From 73dc25761182c74ab3be92611acafe1687c0cd4d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 21 Feb 2025 11:45:53 +0100
Subject: [PATCH 1/2] [VPlan] Construct immutable VPIRBBs for exit blocks at
construction(NFC)
Constract immutable VPIRBasicBlocks for all exit blocks up front and
keep a list of them. Same as the scalar header, they are leaf nodes of
the VPlan and won't change. Some exit blocks may be unreachable, e.g.
if the scalar epilogue always executes or depending on optimizations.
This simplifies both the way we retrieve the exit blocks as well as
hooking up the exit blocks.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++++
llvm/lib/Transforms/Vectorize/VPlan.cpp | 13 ++++++++++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 17 ++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlanCFG.h | 9 ---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 +++------------
5 files changed, 31 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0bf2d71a63ef4..226fc23888f02 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9132,6 +9132,10 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
VPlan &Plan) {
SetVector<VPIRInstruction *> ExitUsersToFix;
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
+ // Nothing to do for unreachable exit blocks.
+ if (ExitVPBB->getNumPredecessors() == 0)
+ continue;
+
for (VPRecipeBase &R : *ExitVPBB) {
auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
if (!ExitIRI)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 121678b52a9f1..fffe544963b11 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -850,6 +850,11 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
VPlan::VPlan(Loop *L) {
setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
ScalarHeader = createVPIRBasicBlock(L->getHeader());
+
+ SmallVector<BasicBlock *> IRExitBlocks;
+ L->getExitBlocks(IRExitBlocks);
+ for (BasicBlock *EB : IRExitBlocks)
+ ExitBlocks.push_back(createVPIRBasicBlock(EB));
}
VPlan::~VPlan() {
@@ -931,7 +936,7 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
// we unconditionally branch to the scalar preheader. Do nothing.
// 3) Otherwise, construct a runtime check.
BasicBlock *IRExitBlock = TheLoop->getUniqueLatchExitBlock();
- auto *VPExitBlock = Plan->createVPIRBasicBlock(IRExitBlock);
+ VPIRBasicBlock *VPExitBlock = Plan->getExitBlock(IRExitBlock);
// The connection order corresponds to the operands of the conditional branch.
VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
@@ -983,6 +988,12 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
}
}
+VPIRBasicBlock *VPlan::getExitBlock(BasicBlock *IRBB) const {
+ return *find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) {
+ return VPIRBB->getIRBasicBlock() == IRBB;
+ });
+}
+
bool VPlan::isExitBlock(VPBlockBase *VPBB) {
return isa<VPIRBasicBlock>(VPBB) && VPBB->getNumSuccessors() == 0;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d86914f0fb026..a964b7f2e935f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3426,6 +3426,11 @@ class VPlan {
/// VPIRBasicBlock wrapping the header of the original scalar loop.
VPIRBasicBlock *ScalarHeader;
+ /// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original
+ /// scalar loop. Note that some exit blocks may be unreachable, e.g. if the
+ /// scalar epilogue always executes
+ SmallVector<VPIRBasicBlock *, 2> ExitBlocks;
+
/// Holds the VFs applicable to this VPlan.
SmallSetVector<ElementCount, 2> VFs;
@@ -3559,11 +3564,13 @@ class VPlan {
/// Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; }
- /// Return an iterator range over the VPIRBasicBlock wrapping the exit blocks
- /// of the VPlan, that is leaf nodes except the scalar header. Defined in
- /// VPlanHCFG, as the definition of the type needs access to the definitions
- /// of VPBlockShallowTraversalWrapper.
- auto getExitBlocks();
+ /// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of
+ /// the original scalar loop.
+ ArrayRef<VPIRBasicBlock *> getExitBlocks() const { return ExitBlocks; }
+
+ /// Return the VPIRBasicBlock corresponding to \p IRBB. \p IRBB must be an
+ /// exit block.
+ VPIRBasicBlock *getExitBlock(BasicBlock *IRBB) const;
/// Returns true if \p VPBB is an exit block.
bool isExitBlock(VPBlockBase *VPBB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanCFG.h b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
index 8fbdacd1ea771..a1014c398789f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanCFG.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
@@ -307,15 +307,6 @@ template <> struct GraphTraits<VPlan *> {
}
};
-inline auto VPlan::getExitBlocks() {
- VPBlockBase *ScalarHeader = getScalarHeader();
- return make_filter_range(
- VPBlockUtils::blocksOnly<VPIRBasicBlock>(
- vp_depth_first_shallow(getVectorLoopRegion()->getSingleSuccessor())),
- [ScalarHeader](VPIRBasicBlock *VPIRBB) {
- return VPIRBB != ScalarHeader && VPIRBB->getNumSuccessors() == 0;
- });
-}
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANCFG_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8e773272300b1..13ef3029023f1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2049,18 +2049,9 @@ void VPlanTransforms::handleUncountableEarlyExit(
cast<BranchInst>(UncountableExitingBlock->getTerminator());
BasicBlock *TrueSucc = EarlyExitingBranch->getSuccessor(0);
BasicBlock *FalseSucc = EarlyExitingBranch->getSuccessor(1);
-
- // The early exit block may or may not be the same as the "countable" exit
- // block. Creates a new VPIRBB for the early exit block in case it is distinct
- // from the countable exit block.
- // TODO: Introduce both exit blocks during VPlan skeleton construction.
- VPIRBasicBlock *VPEarlyExitBlock;
- if (OrigLoop->getUniqueExitBlock()) {
- VPEarlyExitBlock = cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
- } else {
- VPEarlyExitBlock = Plan.createVPIRBasicBlock(
- !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
- }
+ BasicBlock *EarlyExitIRBB =
+ !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc;
+ VPIRBasicBlock *VPEarlyExitBlock = Plan.getExitBlock(EarlyExitIRBB);
VPValue *EarlyExitNotTakenCond = RecipeBuilder.getBlockInMask(
OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
>From d35f56a16a7b42d7262a7c9e7868a1265ce492b2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 25 Feb 2025 11:46:42 +0000
Subject: [PATCH 2/2] !fixup address latest comments, thanks!
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 +++-
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index fffe544963b11..d3c195d4a70ea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -989,9 +989,11 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
}
VPIRBasicBlock *VPlan::getExitBlock(BasicBlock *IRBB) const {
- return *find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) {
+ auto Iter = find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) {
return VPIRBB->getIRBasicBlock() == IRBB;
});
+ assert(Iter != getExitBlocks().end() && "no exit block found");
+ return *Iter;
}
bool VPlan::isExitBlock(VPBlockBase *VPBB) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a964b7f2e935f..1f1af7f87e554 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3427,8 +3427,8 @@ class VPlan {
VPIRBasicBlock *ScalarHeader;
/// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original
- /// scalar loop. Note that some exit blocks may be unreachable, e.g. if the
- /// scalar epilogue always executes
+ /// scalar loop. Note that some exit blocks may be unreachable at the moment,
+ /// e.g. if the scalar epilogue always executes.
SmallVector<VPIRBasicBlock *, 2> ExitBlocks;
/// Holds the VFs applicable to this VPlan.
More information about the llvm-commits
mailing list