[llvm] [LoopVectorize] Address comments on PR #107004 left post-commit (PR #109300)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 23 03:53:40 PDT 2024
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/109300
>From 7dad4adbb02d6f87550d0aadf4ae7ad275445759 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Thu, 19 Sep 2024 15:26:55 +0000
Subject: [PATCH 1/2] [LoopVectorize] Address comments on PR #107004 left
post-commit
* Rename Speculative -> Uncountable and update tests.
* Add comments explaining why it's safe to ignore the predicates
when building up a list of exiting blocks.
* Reshuffle some code to do (hopefully) cheaper checks first.
---
.../Vectorize/LoopVectorizationLegality.h | 22 +++---
.../Vectorize/LoopVectorizationLegality.cpp | 67 ++++++++++---------
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../LoopVectorize/simple_early_exit.ll | 15 ++---
4 files changed, 54 insertions(+), 52 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 091061442ae120..f5b91919a96927 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -377,19 +377,19 @@ class LoopVectorizationLegality {
return LAI->getDepChecker().getMaxSafeVectorWidthInBits();
}
- /// Returns true if the loop has a speculative early exit, i.e. an
+ /// Returns true if the loop has an uncountable early exit, i.e. an
/// uncountable exit that isn't the latch block.
- bool hasSpeculativeEarlyExit() const { return HasSpeculativeEarlyExit; }
+ bool hasUncountableEarlyExit() const { return HasUncountableEarlyExit; }
- /// Returns the speculative early exiting block.
- BasicBlock *getSpeculativeEarlyExitingBlock() const {
+ /// Returns the uncountable early exiting block.
+ BasicBlock *getUncountableEarlyExitingBlock() const {
assert(getUncountableExitingBlocks().size() == 1 &&
"Expected only a single uncountable exiting block");
return getUncountableExitingBlocks()[0];
}
- /// Returns the destination of a speculative early exiting block.
- BasicBlock *getSpeculativeEarlyExitBlock() const {
+ /// Returns the destination of an uncountable early exiting block.
+ BasicBlock *getUncountableEarlyExitBlock() const {
assert(getUncountableExitBlocks().size() == 1 &&
"Expected only a single uncountable exit block");
return getUncountableExitBlocks()[0];
@@ -603,15 +603,17 @@ class LoopVectorizationLegality {
/// the use of those function variants.
bool VecCallVariantsFound = false;
- /// Indicates whether this loop has a speculative early exit, i.e. an
+ /// Indicates whether this loop has an uncountable early exit, i.e. an
/// uncountable exiting block that is not the latch.
- bool HasSpeculativeEarlyExit = false;
+ bool HasUncountableEarlyExit = false;
- /// Keep track of all the loop exiting blocks.
+ /// Keep track of all the countable and uncountable exiting blocks if
+ /// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
SmallVector<BasicBlock *, 4> UncountableExitingBlocks;
- /// Keep track of the destinations of all uncountable exits.
+ /// Keep track of the destinations of all uncountable exits if the
+ /// exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> UncountableExitBlocks;
};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index b767372a56b914..e695902c9d72ad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1473,13 +1473,13 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// Keep a record of all the exiting blocks.
SmallVector<const SCEVPredicate *, 4> Predicates;
- for (BasicBlock *BB1 : ExitingBlocks) {
+ for (BasicBlock *BB : ExitingBlocks) {
const SCEV *EC =
- PSE.getSE()->getPredicatedExitCount(TheLoop, BB1, &Predicates);
+ PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
if (isa<SCEVCouldNotCompute>(EC)) {
- UncountableExitingBlocks.push_back(BB1);
+ UncountableExitingBlocks.push_back(BB);
- SmallVector<BasicBlock *, 2> Succs(successors(BB1));
+ SmallVector<BasicBlock *, 2> Succs(successors(BB));
if (Succs.size() != 2) {
reportVectorizationFailure(
"Early exiting block does not have exactly two successors",
@@ -1488,17 +1488,21 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
return false;
}
- BasicBlock *BB2;
+ BasicBlock *ExitBlock;
if (!TheLoop->contains(Succs[0]))
- BB2 = Succs[0];
+ ExitBlock = Succs[0];
else {
assert(!TheLoop->contains(Succs[1]));
- BB2 = Succs[1];
+ ExitBlock = Succs[1];
}
- UncountableExitBlocks.push_back(BB2);
+ UncountableExitBlocks.push_back(ExitBlock);
} else
- CountableExitingBlocks.push_back(BB1);
+ CountableExitingBlocks.push_back(BB);
}
+ // We can safely ignore the predicates here because when vectorizing the loop
+ // the PredicatatedScalarEvolution class will keep track of all predicates
+ // for each exiting block anyway. This happens when calling
+ // PSE.getSymbolicMaxBackedgeTakenCount() below.
Predicates.clear();
// We only support one uncountable early exit.
@@ -1513,13 +1517,25 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// The only supported early exit loops so far are ones where the early
// exiting block is a unique predecessor of the latch block.
BasicBlock *LatchPredBB = LatchBB->getUniquePredecessor();
- if (LatchPredBB != getSpeculativeEarlyExitingBlock()) {
+ if (LatchPredBB != getUncountableEarlyExitingBlock()) {
reportVectorizationFailure("Early exit is not the latch predecessor",
"Cannot vectorize early exit loop",
"EarlyExitNotLatchPredecessor", ORE, TheLoop);
return false;
}
+ // The latch block must have a countable exit.
+ if (isa<SCEVCouldNotCompute>(
+ PSE.getSE()->getPredicatedExitCount(TheLoop, LatchBB, &Predicates))) {
+ reportVectorizationFailure(
+ "Cannot determine exact exit count for latch block",
+ "Cannot vectorize early exit loop",
+ "UnknownLatchExitCountEarlyExitLoop", ORE, TheLoop);
+ return false;
+ }
+ assert(llvm::is_contained(CountableExitingBlocks, LatchBB) &&
+ "Latch block not found in list of countable exits!");
+
// Check to see if there are instructions that could potentially generate
// exceptions or have side-effects.
auto IsSafeOperation = [](Instruction *I) -> bool {
@@ -1555,18 +1571,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
}
- // The latch block must have a countable exit.
- if (isa<SCEVCouldNotCompute>(
- PSE.getSE()->getPredicatedExitCount(TheLoop, LatchBB, &Predicates))) {
- reportVectorizationFailure(
- "Cannot determine exact exit count for latch block",
- "Cannot vectorize early exit loop",
- "UnknownLatchExitCountEarlyExitLoop", ORE, TheLoop);
- return false;
- }
-
// The vectoriser cannot handle loads that occur after the early exit block.
- assert(LatchBB->getUniquePredecessor() == getSpeculativeEarlyExitingBlock() &&
+ assert(LatchBB->getUniquePredecessor() == getUncountableEarlyExitingBlock() &&
"Expected latch predecessor to be the early exiting block");
// TODO: Handle loops that may fault.
@@ -1580,16 +1586,15 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
return false;
}
- LLVM_DEBUG(
- dbgs()
- << "LV: Found an early exit. Retrying with speculative exit count.\n");
- [[maybe_unused]] const SCEV *SpecExitCount =
+ [[maybe_unused]] const SCEV *SymbolicMaxBTC =
PSE.getSymbolicMaxBackedgeTakenCount();
- assert(!isa<SCEVCouldNotCompute>(SpecExitCount) &&
+ // Since we have an exact exit count for the latch and the early exit
+ // dominates the latch, then this should guarantee a computed SCEV value.
+ assert(!isa<SCEVCouldNotCompute>(SymbolicMaxBTC) &&
"Failed to get symbolic expression for backedge taken count");
-
- LLVM_DEBUG(dbgs() << "LV: Found speculative backedge taken count: "
- << *SpecExitCount << '\n');
+ LLVM_DEBUG(dbgs() << "LV: Found an early exit loop with symbolic max "
+ "backedge taken count: "
+ << *SymbolicMaxBTC << '\n');
return true;
}
@@ -1653,7 +1658,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- HasSpeculativeEarlyExit = false;
+ HasUncountableEarlyExit = false;
if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
if (!isVectorizableEarlyExitLoop()) {
if (DoExtraAnalysis)
@@ -1661,7 +1666,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
else
return false;
} else
- HasSpeculativeEarlyExit = true;
+ HasUncountableEarlyExit = true;
}
// Go over each instruction and look at memory deps.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9685e7d124b7d1..1320c722df7864 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9792,7 +9792,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
return false;
}
- if (LVL.hasSpeculativeEarlyExit()) {
+ if (LVL.hasUncountableEarlyExit()) {
reportVectorizationFailure(
"Auto-vectorization of early exit loops is not yet supported.",
"Auto-vectorization of early exit loops is not yet supported.",
diff --git a/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll
index 936c07b4853a38..47a2318428b055 100644
--- a/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll
@@ -7,8 +7,7 @@ declare void @init_mem(ptr, i64);
define i64 @same_exit_block_pre_inc_use1() {
; DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
-; DEBUG: LV: Found an early exit. Retrying with speculative exit count.
-; DEBUG-NEXT: LV: Found speculative backedge taken count: 63
+; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
; DEBUG-NEXT: LV: We can vectorize this loop!
; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1() {
@@ -1089,8 +1088,7 @@ loop.end:
define i64 @loop_contains_safe_call() {
; DEBUG-LABEL: LV: Checking a loop in 'loop_contains_safe_call'
-; DEBUG: LV: Found an early exit. Retrying with speculative exit count.
-; DEBUG-NEXT: LV: Found speculative backedge taken count: 63
+; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
; DEBUG-NEXT: LV: We can vectorize this loop!
; CHECK-LABEL: define i64 @loop_contains_safe_call() {
; CHECK-NEXT: entry:
@@ -1193,8 +1191,7 @@ loop.end:
define i64 @loop_contains_safe_div() {
; DEBUG-LABEL: LV: Checking a loop in 'loop_contains_safe_div'
-; DEBUG: LV: Found an early exit. Retrying with speculative exit count.
-; DEBUG-NEXT: LV: Found speculative backedge taken count: 63
+; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
; DEBUG-NEXT: LV: We can vectorize this loop!
; CHECK-LABEL: define i64 @loop_contains_safe_div() {
; CHECK-NEXT: entry:
@@ -1347,8 +1344,7 @@ loop.end:
define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(8) %p2) {
; DEBUG-LABEL: LV: Checking a loop in 'loop_contains_load_after_early_exit'
-; DEBUG: LV: Found an early exit. Retrying with speculative exit count.
-; DEBUG-NEXT: LV: Found speculative backedge taken count: 63
+; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
; DEBUG-NEXT: LV: We can vectorize this loop!
; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
; CHECK-LABEL: define i64 @loop_contains_load_after_early_exit(
@@ -1695,8 +1691,7 @@ declare void @abort()
; early is loop invariant.
define i32 @diff_blocks_invariant_early_exit_cond(ptr %s) {
; DEBUG-LABEL: LV: Checking a loop in 'diff_blocks_invariant_early_exit_cond'
-; DEBUG: LV: Found an early exit. Retrying with speculative exit count.
-; DEBUG-NEXT: LV: Found speculative backedge taken count: 275
+; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 275
; DEBUG: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
; CHECK-LABEL: define i32 @diff_blocks_invariant_early_exit_cond(
; CHECK-SAME: ptr [[S:%.*]]) {
>From 3b890b6e1f2c3312be1a35334e9c838a40fefa97 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 23 Sep 2024 10:52:31 +0000
Subject: [PATCH 2/2] Address review comment
* Update diff_exit_block_needs_scev_check test after rebase.
* Update debug/report message when vectoriser bails out for
uncountable early exit loops.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 +++++----
.../Transforms/LoopVectorize/simple_early_exit.ll | 11 +++++------
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1320c722df7864..b181d17c975a85 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9793,10 +9793,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
}
if (LVL.hasUncountableEarlyExit()) {
- reportVectorizationFailure(
- "Auto-vectorization of early exit loops is not yet supported.",
- "Auto-vectorization of early exit loops is not yet supported.",
- "EarlyExitLoopsUnsupported", ORE, L);
+ reportVectorizationFailure("Auto-vectorization of loops with uncountable "
+ "early exit is not yet supported",
+ "Auto-vectorization of loops with uncountable "
+ "early exit is not yet supported",
+ "UncountableEarlyExitLoopsUnsupported", ORE, L);
return false;
}
diff --git a/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll
index 47a2318428b055..49454ae18db79d 100644
--- a/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/simple_early_exit.ll
@@ -9,7 +9,7 @@ define i64 @same_exit_block_pre_inc_use1() {
; DEBUG-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
; DEBUG-NEXT: LV: We can vectorize this loop!
-; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
+; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of loops with uncountable early exit is not yet supported.
; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1() {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
@@ -1346,7 +1346,7 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
; DEBUG-LABEL: LV: Checking a loop in 'loop_contains_load_after_early_exit'
; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63
; DEBUG-NEXT: LV: We can vectorize this loop!
-; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
+; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of loops with uncountable early exit is not yet supported.
; CHECK-LABEL: define i64 @loop_contains_load_after_early_exit(
; CHECK-SAME: ptr align 8 dereferenceable(1024) [[P2:%.*]]) {
; CHECK-NEXT: entry:
@@ -1619,10 +1619,9 @@ loop.end:
; The form of the induction variables requires SCEV predicates.
define i32 @diff_exit_block_needs_scev_check(i32 %end) {
; DEBUG-LABEL: LV: Checking a loop in 'diff_exit_block_needs_scev_check'
-; DEBUG: LV: Found an early exit. Retrying with speculative exit count.
-; DEBUG-NEXT: LV: Found speculative backedge taken count: (-1 + (1 umax (zext i10 (trunc i32 %end to i10) to i32)))<nsw>
+; DEBUG: Found an early exit loop with symbolic max backedge taken count: (-1 + (1 umax (zext i10 (trunc i32 %end to i10) to i32)))<nsw>
; DEBUG-NEXT: LV: We can vectorize this loop!
-; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
+; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of loops with uncountable early exit is not yet supported.
; CHECK-LABEL: define i32 @diff_exit_block_needs_scev_check(
; CHECK-SAME: i32 [[END:%.*]]) {
; CHECK-NEXT: entry:
@@ -1692,7 +1691,7 @@ declare void @abort()
define i32 @diff_blocks_invariant_early_exit_cond(ptr %s) {
; DEBUG-LABEL: LV: Checking a loop in 'diff_blocks_invariant_early_exit_cond'
; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 275
-; DEBUG: LV: Not vectorizing: Auto-vectorization of early exit loops is not yet supported.
+; DEBUG: LV: Not vectorizing: Auto-vectorization of loops with uncountable early exit is not yet supported.
; CHECK-LABEL: define i32 @diff_blocks_invariant_early_exit_cond(
; CHECK-SAME: ptr [[S:%.*]]) {
; CHECK-NEXT: entry:
More information about the llvm-commits
mailing list