[llvm] [LoopVectorize] Use new getUniqueLatchExitBlock routine (PR #108231)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 5 03:31:11 PST 2024
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/108231
>From 696205947f5868ba7abb8383c80c3a80e99eda2b Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 7 Oct 2024 13:24:09 +0000
Subject: [PATCH 1/2] [LoopVectorize] Use new getUniqueLatchExitBlock routine
With PR #88385 I am introducing support for vectorising more
loops with early exits that don't require a scalar epilogue.
As such, if a loop doesn't have a unique exit block it will
not automatically imply we require a scalar epilogue. Also,
in the only place in the code today where we use the variable
LoopExitBlock we actually mean the exit block from the latch.
Therefore, it seemed reasonable to add a new
getUniqueLatchExitBlock helper that allows the caller to
determine the exit block taken from the latch and use this
instead of getUniqueExitBlock. I also removed LoopExitBlock
since it was only used in one place.
While doing this I also noticed that one of the comments in
requiresScalarEpilogue is wrong when we require a scalar
epilogue, i.e. when we're not exiting from the latch block.
This doesn't always imply we have multiple exits, e.g. see
the test in
Transforms/LoopVectorize/unroll_nonlatch.ll
where the latch unconditionally branches back to the only
exiting block.
---
llvm/include/llvm/Support/GenericLoopInfo.h | 4 ++++
.../llvm/Support/GenericLoopInfoImpl.h | 10 ++++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 19 +++++++------------
3 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/Support/GenericLoopInfo.h b/llvm/include/llvm/Support/GenericLoopInfo.h
index d560ca648132c9..0fa13e2a3d0e15 100644
--- a/llvm/include/llvm/Support/GenericLoopInfo.h
+++ b/llvm/include/llvm/Support/GenericLoopInfo.h
@@ -294,6 +294,10 @@ template <class BlockT, class LoopT> class LoopBase {
/// Otherwise return null.
BlockT *getUniqueExitBlock() const;
+ /// Return the unique exit block for the latch, or null if there are multiple
+ /// different exit blocks.
+ BlockT *getUniqueLatchExitBlock() const;
+
/// Return true if this loop does not have any exit blocks.
bool hasNoExitBlocks() const;
diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
index d19022729ace32..4945ea30950d23 100644
--- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h
+++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
@@ -159,6 +159,16 @@ BlockT *LoopBase<BlockT, LoopT>::getUniqueExitBlock() const {
return getExitBlockHelper(this, true).first;
}
+template <class BlockT, class LoopT>
+BlockT *LoopBase<BlockT, LoopT>::getUniqueLatchExitBlock() const {
+ const BlockT *Latch = getLoopLatch();
+ assert(Latch && "Latch block must exists");
+ SmallVector<BlockT *, 4> ExitBlocks;
+ getUniqueExitBlocksHelper(this, ExitBlocks,
+ [Latch](const BlockT *BB) { return BB == Latch; });
+ return ExitBlocks.size() == 1 ? ExitBlocks[0] : nullptr;
+}
+
/// getExitEdges - Return all pairs of (_inside_block_,_outside_block_).
template <class BlockT, class LoopT>
void LoopBase<BlockT, LoopT>::getExitEdges(
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 58fcba93f1a188..9e64d2b8de7bfc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -634,10 +634,6 @@ class InnerLoopVectorizer {
/// Middle Block between the vector and the scalar.
BasicBlock *LoopMiddleBlock;
- /// The unique ExitBlock of the scalar loop if one exists. Note that
- /// there can be multiple exiting edges reaching this block.
- BasicBlock *LoopExitBlock;
-
/// A list of all bypass blocks. The first block is the entry of the loop.
SmallVector<BasicBlock *, 4> LoopBypassBlocks;
@@ -1365,8 +1361,8 @@ class LoopVectorizationCostModel {
// If we might exit from anywhere but the latch, must run the exiting
// iteration in scalar form.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
- LLVM_DEBUG(
- dbgs() << "LV: Loop requires scalar epilogue: multiple exits\n");
+ LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
+ "from latch block\n");
return true;
}
if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
@@ -2025,8 +2021,7 @@ class GeneratedRTChecks {
/// adjusts the branches to branch to the vector preheader or \p Bypass,
/// depending on the generated condition.
BasicBlock *emitSCEVChecks(BasicBlock *Bypass,
- BasicBlock *LoopVectorPreHeader,
- BasicBlock *LoopExitBlock) {
+ BasicBlock *LoopVectorPreHeader) {
if (!SCEVCheckCond)
return nullptr;
@@ -2522,7 +2517,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
BasicBlock *const SCEVCheckBlock =
- RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader, LoopExitBlock);
+ RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader);
if (!SCEVCheckBlock)
return nullptr;
@@ -2576,8 +2571,8 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
LoopVectorPreHeader = OrigLoop->getLoopPreheader();
assert(LoopVectorPreHeader && "Invalid loop structure");
- LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr
- assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF.isVector())) &&
+ assert((OrigLoop->getUniqueExitBlock() ||
+ Cost->requiresScalarEpilogue(VF.isVector())) &&
"multiple exit loop without required epilogue?");
LoopMiddleBlock =
@@ -7927,7 +7922,7 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
// If there is an epilogue which must run, there's no edge from the
// middle block to exit blocks and thus no need to update the immediate
// dominator of the exit blocks.
- DT->changeImmediateDominator(LoopExitBlock,
+ DT->changeImmediateDominator(OrigLoop->getUniqueLatchExitBlock(),
EPI.EpilogueIterationCountCheck);
// Keep track of bypass blocks, as they feed start values to the induction and
>From 35e3a47e2ab3b06fd8d5e948d110e0f502f66351 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Tue, 5 Nov 2024 11:30:21 +0000
Subject: [PATCH 2/2] Address review comment
* Use find_singleton in getUniqueLatchExitBlock.
---
llvm/include/llvm/Support/GenericLoopInfoImpl.h | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
index 4945ea30950d23..fc43cd7f388f4d 100644
--- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h
+++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
@@ -161,12 +161,13 @@ BlockT *LoopBase<BlockT, LoopT>::getUniqueExitBlock() const {
template <class BlockT, class LoopT>
BlockT *LoopBase<BlockT, LoopT>::getUniqueLatchExitBlock() const {
- const BlockT *Latch = getLoopLatch();
+ BlockT *Latch = getLoopLatch();
assert(Latch && "Latch block must exists");
- SmallVector<BlockT *, 4> ExitBlocks;
- getUniqueExitBlocksHelper(this, ExitBlocks,
- [Latch](const BlockT *BB) { return BB == Latch; });
- return ExitBlocks.size() == 1 ? ExitBlocks[0] : nullptr;
+ auto IsExitBlock = [&](BlockT *BB, bool AllowRepeats) -> BlockT * {
+ assert(!AllowRepeats && "Unexpected parameter value.");
+ return !contains(BB) ? BB : nullptr;
+ };
+ return find_singleton<BlockT>(children<BlockT *>(Latch), IsExitBlock);
}
/// getExitEdges - Return all pairs of (_inside_block_,_outside_block_).
More information about the llvm-commits
mailing list