[llvm] [LV][NFC] Refactor structures used to maintain uncountable exit info (PR #123219)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 20 02:13:08 PST 2025
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/123219
>From 3e474b1d7976026c46949e813596d040e363178e Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Thu, 16 Jan 2025 16:45:06 +0000
Subject: [PATCH 1/4] [LV][NFC] Refactor structures used to maintain
uncountable exit info
I've removed the HasUncountableEarlyExit variable, since we can
already determine whether or not a loop has an early exit by seeing
how many uncountable exit blocks we found.
I have also deleted the old UncountableExitingBlocks and
UncountableExitBlocks lists and replaced them with a single list of
edges. This means we don't need to worry about keeping the list
entries in sync and makes it clear which exiting block corresponds
to which exit block.
---
.../Vectorize/LoopVectorizationLegality.h | 40 +++++++------------
.../Vectorize/LoopVectorizationLegality.cpp | 11 ++---
2 files changed, 18 insertions(+), 33 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 72fda911962ad2..5764e3a487f3e3 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -391,25 +391,24 @@ class LoopVectorizationLegality {
/// Returns true if the loop has an uncountable early exit, i.e. an
/// uncountable exit that isn't the latch block.
- bool hasUncountableEarlyExit() const { return HasUncountableEarlyExit; }
+ bool hasUncountableEarlyExit() const { return getUncountableEdges().size(); }
/// Returns the uncountable early exiting block.
BasicBlock *getUncountableEarlyExitingBlock() const {
- if (!HasUncountableEarlyExit) {
- assert(getUncountableExitingBlocks().empty() &&
- "Expected no uncountable exiting blocks");
+ if (!hasUncountableEarlyExit())
return nullptr;
- }
- assert(getUncountableExitingBlocks().size() == 1 &&
+ assert(getUncountableEdges().size() == 1 &&
"Expected only a single uncountable exiting block");
- return getUncountableExitingBlocks()[0];
+ return getUncountableEdges()[0].first;
}
/// Returns the destination of an uncountable early exiting block.
BasicBlock *getUncountableEarlyExitBlock() const {
- assert(getUncountableExitBlocks().size() == 1 &&
+ if (!hasUncountableEarlyExit())
+ return nullptr;
+ assert(getUncountableEdges().size() == 1 &&
"Expected only a single uncountable exit block");
- return getUncountableExitBlocks()[0];
+ return getUncountableEdges()[0].second;
}
/// Returns true if vector representation of the instruction \p I
@@ -463,14 +462,10 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
- /// Returns all the exiting blocks with an uncountable exit.
- const SmallVector<BasicBlock *, 4> &getUncountableExitingBlocks() const {
- return UncountableExitingBlocks;
- }
-
- /// Returns all the exit blocks from uncountable exiting blocks.
- SmallVector<BasicBlock *, 4> getUncountableExitBlocks() const {
- return UncountableExitBlocks;
+ /// Returns all the loop edges that have an uncountable exit.
+ const SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> &
+ getUncountableEdges() const {
+ return UncountableEdges;
}
private:
@@ -654,18 +649,13 @@ class LoopVectorizationLegality {
/// supported.
bool StructVecCallFound = false;
- /// Indicates whether this loop has an uncountable early exit, i.e. an
- /// uncountable exiting block that is not the latch.
- bool HasUncountableEarlyExit = false;
-
/// Keep track of all the countable and uncountable exiting blocks if
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
- SmallVector<BasicBlock *, 4> UncountableExitingBlocks;
- /// Keep track of the destinations of all uncountable exits if the
- /// exact backedge taken count is not computable.
- SmallVector<BasicBlock *, 4> UncountableExitBlocks;
+ /// Keep track of all the loop edges with uncountable exits, where each entry
+ /// is a pair of (Exiting, Exit) blocks.
+ SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> UncountableEdges;
};
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 406864a6793dc8..00bd429a0ef9d5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1635,8 +1635,6 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
const SCEV *EC =
PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
if (isa<SCEVCouldNotCompute>(EC)) {
- UncountableExitingBlocks.push_back(BB);
-
SmallVector<BasicBlock *, 2> Succs(successors(BB));
if (Succs.size() != 2) {
reportVectorizationFailure(
@@ -1653,7 +1651,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
assert(!TheLoop->contains(Succs[1]));
ExitBlock = Succs[1];
}
- UncountableExitBlocks.push_back(ExitBlock);
+ UncountableEdges.push_back({BB, ExitBlock});
} else
CountableExitingBlocks.push_back(BB);
}
@@ -1664,7 +1662,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
Predicates.clear();
// We only support one uncountable early exit.
- if (getUncountableExitingBlocks().size() != 1) {
+ if (getUncountableEdges().size() != 1) {
reportVectorizationFailure(
"Loop has too many uncountable exits",
"Cannot vectorize early exit loop with more than one early exit",
@@ -1812,7 +1810,6 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- HasUncountableEarlyExit = false;
if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
if (TheLoop->getExitingBlock()) {
reportVectorizationFailure("Cannot vectorize uncountable loop",
@@ -1822,10 +1819,8 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
else
return false;
} else {
- HasUncountableEarlyExit = true;
if (!isVectorizableEarlyExitLoop()) {
- UncountableExitingBlocks.clear();
- HasUncountableEarlyExit = false;
+ UncountableEdges.clear();
if (DoExtraAnalysis)
Result = false;
else
>From 1323c0671e44f6a7c410c350c0f3169437636372 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 17 Jan 2025 09:47:20 +0000
Subject: [PATCH 2/4] Address review comments
---
.../Vectorize/LoopVectorizationLegality.h | 26 +++++++++----------
.../Vectorize/LoopVectorizationLegality.cpp | 10 ++++---
2 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 5764e3a487f3e3..a8a2695b5169ae 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -391,24 +391,22 @@ class LoopVectorizationLegality {
/// Returns true if the loop has an uncountable early exit, i.e. an
/// uncountable exit that isn't the latch block.
- bool hasUncountableEarlyExit() const { return getUncountableEdges().size(); }
+ bool hasUncountableEarlyExit() const {
+ return getUncountableEdge().has_value();
+ }
/// Returns the uncountable early exiting block.
BasicBlock *getUncountableEarlyExitingBlock() const {
if (!hasUncountableEarlyExit())
return nullptr;
- assert(getUncountableEdges().size() == 1 &&
- "Expected only a single uncountable exiting block");
- return getUncountableEdges()[0].first;
+ return (*getUncountableEdge()).first;
}
/// Returns the destination of an uncountable early exiting block.
BasicBlock *getUncountableEarlyExitBlock() const {
if (!hasUncountableEarlyExit())
return nullptr;
- assert(getUncountableEdges().size() == 1 &&
- "Expected only a single uncountable exit block");
- return getUncountableEdges()[0].second;
+ return (*getUncountableEdge()).second;
}
/// Returns true if vector representation of the instruction \p I
@@ -462,10 +460,10 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
- /// Returns all the loop edges that have an uncountable exit.
- const SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> &
- getUncountableEdges() const {
- return UncountableEdges;
+ /// Returns the loop edge with an uncountable exit.
+ std::optional<std::pair<BasicBlock *, BasicBlock *>>
+ getUncountableEdge() const {
+ return UncountableEdge;
}
private:
@@ -653,9 +651,9 @@ class LoopVectorizationLegality {
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
- /// Keep track of all the loop edges with uncountable exits, where each entry
- /// is a pair of (Exiting, Exit) blocks.
- SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> UncountableEdges;
+ /// Keep track of the loop edge with an uncountable exit, comprising a pair
+ /// of (Exiting, Exit) blocks.
+ std::optional<std::pair<BasicBlock *, BasicBlock *>> UncountableEdge;
};
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 00bd429a0ef9d5..f3a44a0414985e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1631,6 +1631,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// Keep a record of all the exiting blocks.
SmallVector<const SCEVPredicate *, 4> Predicates;
+ SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> UncountableEdges;
for (BasicBlock *BB : ExitingBlocks) {
const SCEV *EC =
PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
@@ -1662,7 +1663,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
Predicates.clear();
// We only support one uncountable early exit.
- if (getUncountableEdges().size() != 1) {
+ if (UncountableEdges.size() != 1) {
reportVectorizationFailure(
"Loop has too many uncountable exits",
"Cannot vectorize early exit loop with more than one early exit",
@@ -1673,7 +1674,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// The only supported early exit loops so far are ones where the early
// exiting block is a unique predecessor of the latch block.
BasicBlock *LatchPredBB = LatchBB->getUniquePredecessor();
- if (LatchPredBB != getUncountableEarlyExitingBlock()) {
+ if (LatchPredBB != UncountableEdges[0].first) {
reportVectorizationFailure("Early exit is not the latch predecessor",
"Cannot vectorize early exit loop",
"EarlyExitNotLatchPredecessor", ORE, TheLoop);
@@ -1726,7 +1727,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
// The vectoriser cannot handle loads that occur after the early exit block.
- assert(LatchBB->getUniquePredecessor() == getUncountableEarlyExitingBlock() &&
+ assert(LatchBB->getUniquePredecessor() == UncountableEdges[0].first &&
"Expected latch predecessor to be the early exiting block");
// TODO: Handle loops that may fault.
@@ -1749,6 +1750,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
LLVM_DEBUG(dbgs() << "LV: Found an early exit loop with symbolic max "
"backedge taken count: "
<< *SymbolicMaxBTC << '\n');
+ UncountableEdge = UncountableEdges[0];
return true;
}
@@ -1820,7 +1822,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
} else {
if (!isVectorizableEarlyExitLoop()) {
- UncountableEdges.clear();
+ UncountableEdge = std::nullopt;
if (DoExtraAnalysis)
Result = false;
else
>From b6d7019bb57baea6cb286a7577018d5c3f447c2f Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 17 Jan 2025 15:52:52 +0000
Subject: [PATCH 3/4] Address review comments
---
.../Vectorize/LoopVectorizationLegality.h | 20 +++++++++----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index a8a2695b5169ae..721e5c72f44e66 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -389,24 +389,21 @@ class LoopVectorizationLegality {
return LAI->getDepChecker().getMaxSafeVectorWidthInBits();
}
- /// Returns true if the loop has an uncountable early exit, i.e. an
+ /// Returns true if the loop has exactly one uncountable early exit, i.e. an
/// uncountable exit that isn't the latch block.
bool hasUncountableEarlyExit() const {
return getUncountableEdge().has_value();
}
- /// Returns the uncountable early exiting block.
+ /// Returns the uncountable early exiting block, if there is exactly one.
BasicBlock *getUncountableEarlyExitingBlock() const {
- if (!hasUncountableEarlyExit())
- return nullptr;
- return (*getUncountableEdge()).first;
+ return hasUncountableEarlyExit() ? getUncountableEdge()->first : nullptr;
}
- /// Returns the destination of an uncountable early exiting block.
+ /// Returns the destination of the uncountable early exiting block, if there
+ /// is exactly one.
BasicBlock *getUncountableEarlyExitBlock() const {
- if (!hasUncountableEarlyExit())
- return nullptr;
- return (*getUncountableEdge()).second;
+ return hasUncountableEarlyExit() ? getUncountableEdge()->second : nullptr;
}
/// Returns true if vector representation of the instruction \p I
@@ -460,7 +457,8 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
- /// Returns the loop edge with an uncountable exit.
+ /// Returns the loop edge with an uncountable exit, or std::nullopt if there
+ /// isn't a single such edge.
std::optional<std::pair<BasicBlock *, BasicBlock *>>
getUncountableEdge() const {
return UncountableEdge;
@@ -652,7 +650,7 @@ class LoopVectorizationLegality {
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
/// Keep track of the loop edge with an uncountable exit, comprising a pair
- /// of (Exiting, Exit) blocks.
+ /// of (Exiting, Exit) blocks, if there is exactly one early exit.
std::optional<std::pair<BasicBlock *, BasicBlock *>> UncountableEdge;
};
>From 85b5e895b0a7ed60fe45b2fa4dd62f465dbcd724 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 20 Jan 2025 10:12:19 +0000
Subject: [PATCH 4/4] Address review comments
---
.../Vectorize/LoopVectorizationLegality.h | 4 +--
.../Vectorize/LoopVectorizationLegality.cpp | 27 +++++++++++--------
2 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 721e5c72f44e66..3c5cf1ebe6ba2e 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -457,7 +457,7 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
- /// Returns the loop edge with an uncountable exit, or std::nullopt if there
+ /// Returns the loop edge to an uncountable exit, or std::nullopt if there
/// isn't a single such edge.
std::optional<std::pair<BasicBlock *, BasicBlock *>>
getUncountableEdge() const {
@@ -649,7 +649,7 @@ class LoopVectorizationLegality {
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
- /// Keep track of the loop edge with an uncountable exit, comprising a pair
+ /// Keep track of the loop edge to an uncountable exit, comprising a pair
/// of (Exiting, Exit) blocks, if there is exactly one early exit.
std::optional<std::pair<BasicBlock *, BasicBlock *>> UncountableEdge;
};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index f3a44a0414985e..e3599315e224f3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1631,7 +1631,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// Keep a record of all the exiting blocks.
SmallVector<const SCEVPredicate *, 4> Predicates;
- SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> UncountableEdges;
+ std::optional<std::pair<BasicBlock *, BasicBlock *>> SingleUncountableEdge;
for (BasicBlock *BB : ExitingBlocks) {
const SCEV *EC =
PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
@@ -1652,7 +1652,16 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
assert(!TheLoop->contains(Succs[1]));
ExitBlock = Succs[1];
}
- UncountableEdges.push_back({BB, ExitBlock});
+
+ if (SingleUncountableEdge) {
+ reportVectorizationFailure(
+ "Loop has too many uncountable exits",
+ "Cannot vectorize early exit loop with more than one early exit",
+ "TooManyUncountableEarlyExits", ORE, TheLoop);
+ return false;
+ }
+
+ SingleUncountableEdge = {BB, ExitBlock};
} else
CountableExitingBlocks.push_back(BB);
}
@@ -1662,19 +1671,15 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// PSE.getSymbolicMaxBackedgeTakenCount() below.
Predicates.clear();
- // We only support one uncountable early exit.
- if (UncountableEdges.size() != 1) {
- reportVectorizationFailure(
- "Loop has too many uncountable exits",
- "Cannot vectorize early exit loop with more than one early exit",
- "TooManyUncountableEarlyExits", ORE, TheLoop);
+ if (!SingleUncountableEdge) {
+ LLVM_DEBUG(dbgs() << "LV: Cound not find any uncountable exits");
return false;
}
// The only supported early exit loops so far are ones where the early
// exiting block is a unique predecessor of the latch block.
BasicBlock *LatchPredBB = LatchBB->getUniquePredecessor();
- if (LatchPredBB != UncountableEdges[0].first) {
+ if (LatchPredBB != SingleUncountableEdge->first) {
reportVectorizationFailure("Early exit is not the latch predecessor",
"Cannot vectorize early exit loop",
"EarlyExitNotLatchPredecessor", ORE, TheLoop);
@@ -1727,7 +1732,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
// The vectoriser cannot handle loads that occur after the early exit block.
- assert(LatchBB->getUniquePredecessor() == UncountableEdges[0].first &&
+ assert(LatchBB->getUniquePredecessor() == SingleUncountableEdge->first &&
"Expected latch predecessor to be the early exiting block");
// TODO: Handle loops that may fault.
@@ -1750,7 +1755,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
LLVM_DEBUG(dbgs() << "LV: Found an early exit loop with symbolic max "
"backedge taken count: "
<< *SymbolicMaxBTC << '\n');
- UncountableEdge = UncountableEdges[0];
+ UncountableEdge = SingleUncountableEdge;
return true;
}
More information about the llvm-commits
mailing list