[llvm] [LV] Strengthen calls to collectInstsToScalarize (NFC) (PR #130642)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 28 03:52:48 PDT 2025
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/130642
>From fc6dda60c57cb8d3e086e4b61215bc69b2eb8e41 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Mon, 10 Mar 2025 17:23:50 +0000
Subject: [PATCH 1/3] [LV] Strengthen calls to collectInstsToScalarize (NFC)
Avoid the pattern of always calling collectInstsToScalarize after
collectUniformsAndScalars, and call it in collectUniformsAndScalars
instead. Also strengthen checks for early exits in the function.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 25 ++++++++-----------
1 file changed, 10 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 667529c5c3296..0109085f91f4c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -980,7 +980,6 @@ class LoopVectorizationCostModel {
/// \return true if the UserVF is a feasible VF to be chosen.
bool selectUserVectorizationFactor(ElementCount UserVF) {
collectUniformsAndScalars(UserVF);
- collectInstsToScalarize(UserVF);
return expectedCost(UserVF).isValid();
}
@@ -1250,6 +1249,7 @@ class LoopVectorizationCostModel {
collectLoopUniforms(VF);
setVectorizedCallDecision(VF);
collectLoopScalars(VF);
+ collectInstsToScalarize(VF);
}
/// Returns true if the target machine supports masked store operation
@@ -5382,11 +5382,14 @@ bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
}
void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
- // If we aren't vectorizing the loop, or if we've already collected the
- // instructions to scalarize, there's nothing to do. Collection may already
- // have occurred if we have a user-selected VF and are now computing the
- // expected cost for interleaving.
- if (VF.isScalar() || VF.isZero() || InstsToScalarize.contains(VF))
+ assert(VF.isVector() && "Expected VF >= 2");
+
+ // If we've already collected the instructions to scalarize or the predicated
+ // BBs after vectorization, there's nothing to do. Collection may already have
+ // occurred if we have a user-selected VF and are now computing the expected
+ // cost for interleaving.
+ if (InstsToScalarize.contains(VF) ||
+ PredicatedBBsAfterVectorization.contains(VF))
return;
// Initialize a mapping for VF in InstsToScalalarize. If we find that it's
@@ -5394,8 +5397,6 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
// map will indicate that we've analyzed it already.
ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
- PredicatedBBsAfterVectorization[VF].clear();
-
// Find all the instructions that are scalar with predication in the loop and
// determine if it would be better to not if-convert the blocks they are in.
// If so, we also record the instructions to scalarize.
@@ -7185,16 +7186,10 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
VFCandidates.push_back(VF);
CM.collectInLoopReductions();
- for (const auto &VF : VFCandidates) {
+ for (const auto &VF : VFCandidates)
// Collect Uniform and Scalar instructions after vectorization with VF.
CM.collectUniformsAndScalars(VF);
- // Collect the instructions (and their associated costs) that will be more
- // profitable to scalarize.
- if (VF.isVector())
- CM.collectInstsToScalarize(VF);
- }
-
buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);
buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF);
>From bb69f8ffaa25219950f3be942661430af8d07bb6 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Thu, 27 Mar 2025 14:06:52 +0000
Subject: [PATCH 2/3] [LV] Address review
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0109085f91f4c..dd16ea8fcf8d2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -979,7 +979,7 @@ class LoopVectorizationCostModel {
/// Setup cost-based decisions for user vectorization factor.
/// \return true if the UserVF is a feasible VF to be chosen.
bool selectUserVectorizationFactor(ElementCount UserVF) {
- collectUniformsAndScalars(UserVF);
+ collectNonVectorized(UserVF);
return expectedCost(UserVF).isValid();
}
@@ -1235,13 +1235,13 @@ class LoopVectorizationCostModel {
/// the loop.
void collectInstsToScalarize(ElementCount VF);
- /// Collect Uniform and Scalar values for the given \p VF.
+ /// Collect Uniforms, Scalars, and Insts to Scalarize for the given \p VF.
/// The sets depend on CM decision for Load/Store instructions
/// that may be vectorized as interleave, gather-scatter or scalarized.
/// Also make a decision on what to do about call instructions in the loop
/// at that VF -- scalarize, call a known vector routine, or call a
/// vector intrinsic.
- void collectUniformsAndScalars(ElementCount VF) {
+ void collectNonVectorized(ElementCount VF) {
// Do the analysis once.
if (VF.isScalar() || Uniforms.contains(VF))
return;
@@ -5285,7 +5285,7 @@ LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) {
RegUsage[ClassID] += 1;
}
} else {
- collectUniformsAndScalars(VFs[J]);
+ collectNonVectorized(VFs[J]);
for (auto *Inst : OpenIntervals) {
// Skip ignored values for VF > 1.
if (VecValuesToIgnore.count(Inst))
@@ -7186,9 +7186,10 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
VFCandidates.push_back(VF);
CM.collectInLoopReductions();
- for (const auto &VF : VFCandidates)
+ for (const auto &VF : VFCandidates) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- CM.collectUniformsAndScalars(VF);
+ CM.collectNonVectorized(VF);
+ }
buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);
buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF);
>From ebab794ae05f5a7ecb0a3b400d538ad08825ebc7 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Fri, 28 Mar 2025 10:39:22 +0000
Subject: [PATCH 3/3] [LV] Tweak function name, update comment
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dd16ea8fcf8d2..66198c4b48ade 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -979,7 +979,7 @@ class LoopVectorizationCostModel {
/// Setup cost-based decisions for user vectorization factor.
/// \return true if the UserVF is a feasible VF to be chosen.
bool selectUserVectorizationFactor(ElementCount UserVF) {
- collectNonVectorized(UserVF);
+ collectNonVectorizedSetWideningDecisions(UserVF);
return expectedCost(UserVF).isValid();
}
@@ -1235,13 +1235,14 @@ class LoopVectorizationCostModel {
/// the loop.
void collectInstsToScalarize(ElementCount VF);
- /// Collect Uniforms, Scalars, and Insts to Scalarize for the given \p VF.
+ /// Collect values that will not be widened, including Uniforms, Scalars, and
+ /// Instructions to Scalarize for the given \p VF.
/// The sets depend on CM decision for Load/Store instructions
/// that may be vectorized as interleave, gather-scatter or scalarized.
/// Also make a decision on what to do about call instructions in the loop
/// at that VF -- scalarize, call a known vector routine, or call a
/// vector intrinsic.
- void collectNonVectorized(ElementCount VF) {
+ void collectNonVectorizedSetWideningDecisions(ElementCount VF) {
// Do the analysis once.
if (VF.isScalar() || Uniforms.contains(VF))
return;
@@ -5285,7 +5286,7 @@ LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) {
RegUsage[ClassID] += 1;
}
} else {
- collectNonVectorized(VFs[J]);
+ collectNonVectorizedSetWideningDecisions(VFs[J]);
for (auto *Inst : OpenIntervals) {
// Skip ignored values for VF > 1.
if (VecValuesToIgnore.count(Inst))
@@ -7188,7 +7189,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
CM.collectInLoopReductions();
for (const auto &VF : VFCandidates) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- CM.collectNonVectorized(VF);
+ CM.collectNonVectorizedSetWideningDecisions(VF);
}
buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);
More information about the llvm-commits
mailing list