[llvm] [LV][NFC] Factor out vectorization utility functions to a separate file (PR #194951)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 13:51:47 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/194951
>From 695d8fc2c2b664c81b0c50284da4b5943c92a86a Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 29 Apr 2026 12:56:34 +0000
Subject: [PATCH 1/2] add const to Legality functions that just return
values/vars.
Change-Id: I138acab0d872af6717eda0e9f8c2cbe7dcfc2dde
---
.../llvm/Transforms/Vectorize/LoopVectorizationLegality.h | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 49e8bb1e85526..25e68b775c960 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -239,7 +239,7 @@ class LoopVectorizationRequirements {
ExactFPMathInst = I;
}
- Instruction *getExactFPInst() { return ExactFPMathInst; }
+ Instruction *getExactFPInst() const { return ExactFPMathInst; }
private:
Instruction *ExactFPMathInst = nullptr;
@@ -324,7 +324,7 @@ class LoopVectorizationLegality {
void prepareToFoldTailByMasking();
/// Returns the primary induction variable.
- PHINode *getPrimaryInduction() { return PrimaryInduction; }
+ PHINode *getPrimaryInduction() const { return PrimaryInduction; }
/// Returns the reduction variables found in the loop.
const ReductionList &getReductionVars() const { return Reductions; }
@@ -344,7 +344,7 @@ class LoopVectorizationLegality {
RecurrenceSet &getFixedOrderRecurrences() { return FixedOrderRecurrences; }
/// Returns the widest induction type.
- IntegerType *getWidestInductionType() { return WidestIndTy; }
+ IntegerType *getWidestInductionType() const { return WidestIndTy; }
/// Returns True if given store is a final invariant store of one of the
/// reductions found in the loop.
>From 222c90ff089fe5da15b07930dc84e69b03b99884 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 29 Apr 2026 20:36:08 +0000
Subject: [PATCH 2/2] [LV][NFC] Factor out vectorization utility functions to a
separate file
---
.../llvm/Transforms/Vectorize/LoopVectorize.h | 17 -
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
.../Vectorize/LoopVectorizationLegality.cpp | 217 +++++++-----
.../Vectorize/LoopVectorizationPlanner.cpp | 38 +-
.../Vectorize/LoopVectorizationPlanner.h | 10 +-
.../Vectorize/LoopVectorizationUtils.cpp | 132 +++++++
.../Vectorize/LoopVectorizationUtils.h | 103 ++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 328 ++++++------------
8 files changed, 506 insertions(+), 340 deletions(-)
create mode 100644 llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.cpp
create mode 100644 llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.h
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
index 6eab92e66745e..2fe7d7692d64d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -165,23 +165,6 @@ struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> {
LLVM_ABI bool processLoop(Loop *L);
};
-/// Reports a vectorization failure: print \p DebugMsg for debugging
-/// purposes along with the corresponding optimization remark \p RemarkName.
-/// If \p I is passed, it is an instruction that prevents vectorization.
-/// Otherwise, the loop \p TheLoop is used for the location of the remark.
-LLVM_ABI void reportVectorizationFailure(
- const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag,
- OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I = nullptr);
-
-/// Same as above, but the debug message and optimization remark are identical
-inline void reportVectorizationFailure(const StringRef DebugMsg,
- const StringRef ORETag,
- OptimizationRemarkEmitter *ORE,
- Loop *TheLoop,
- Instruction *I = nullptr) {
- reportVectorizationFailure(DebugMsg, DebugMsg, ORETag, ORE, TheLoop, I);
-}
-
/// A marker analysis to determine if extra passes should be run after loop
/// vectorization.
struct ShouldRunExtraVectorPasses
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 984b0d97c60c8..0739f2ad3744b 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -34,6 +34,7 @@ add_llvm_component_library(LLVMVectorize
VPlanUnroll.cpp
VPlanVerifier.cpp
VPlanUtils.cpp
+ LoopVectorizationUtils.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index b2d0c6bb11202..8051849eb3c8a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -15,6 +15,7 @@
//
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
+#include "LoopVectorizationUtils.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -646,7 +647,9 @@ bool LoopVectorizationLegality::canVectorizeOuterLoop() {
// not supported yet.
Instruction *Term = BB->getTerminator();
if (!isa<UncondBrInst, CondBrInst>(Term)) {
- reportVectorizationFailure("Unsupported basic block terminator",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Unsupported basic block terminator",
"loop control flow is not understood by vectorizer",
"CFGNotUnderstood", ORE, TheLoop);
if (DoExtraAnalysis)
@@ -665,7 +668,8 @@ bool LoopVectorizationLegality::canVectorizeOuterLoop() {
if (Br && !TheLoop->isLoopInvariant(Br->getCondition()) &&
!LI->isLoopHeader(Br->getSuccessor(0)) &&
!LI->isLoopHeader(Br->getSuccessor(1))) {
- reportVectorizationFailure("Unsupported conditional branch",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Unsupported conditional branch",
"loop control flow is not understood by vectorizer",
"CFGNotUnderstood", ORE, TheLoop);
if (DoExtraAnalysis)
@@ -679,9 +683,11 @@ bool LoopVectorizationLegality::canVectorizeOuterLoop() {
// simple outer loops scenarios with uniform nested loops.
if (!isUniformLoopNest(TheLoop /*loop nest*/,
TheLoop /*context outer loop*/)) {
- reportVectorizationFailure("Outer loop contains divergent loops",
- "loop control flow is not understood by vectorizer",
- "CFGNotUnderstood", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Outer loop contains divergent loops",
+ "loop control flow is not understood by vectorizer", "CFGNotUnderstood",
+ ORE, TheLoop);
if (DoExtraAnalysis)
Result = false;
else
@@ -690,8 +696,9 @@ bool LoopVectorizationLegality::canVectorizeOuterLoop() {
// Check whether we are able to set up outer loop induction.
if (!setupOuterLoopInductions()) {
- reportVectorizationFailure("Unsupported outer loop Phi(s)",
- "UnsupportedPhi", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Unsupported outer loop Phi(s)",
+ "UnsupportedPhi", ORE, TheLoop);
if (DoExtraAnalysis)
Result = false;
else
@@ -822,14 +829,16 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
if (!PrimaryInduction) {
if (Inductions.empty()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Did not find one integer induction var",
"loop induction variable could not be identified",
"NoInductionVariable", ORE, TheLoop);
return false;
}
if (!WidestIndTy) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Did not find one integer induction var",
"integer loop induction variable could not be identified",
"NoIntegerInductionVariable", ORE, TheLoop);
@@ -856,8 +865,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// Check that this PHI type is allowed.
if (!PhiTy->isIntegerTy() && !PhiTy->isFloatingPointTy() &&
!PhiTy->isPointerTy()) {
- reportVectorizationFailure(
- "Found a non-int non-pointer PHI",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Found a non-int non-pointer PHI",
"loop control flow is not understood by vectorizer",
"CFGNotUnderstood", ORE, TheLoop);
return false;
@@ -878,8 +887,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// We only allow if-converted PHIs with exactly two incoming values.
if (Phi->getNumIncomingValues() != 2) {
- reportVectorizationFailure(
- "Found an invalid PHI",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Found an invalid PHI",
"loop control flow is not understood by vectorizer",
"CFGNotUnderstood", ORE, TheLoop, Phi);
return false;
@@ -947,11 +956,11 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
return true;
}
- reportVectorizationFailure("Found an unidentified PHI",
- "value that could not be identified as "
- "reduction is used outside the loop",
- "NonReductionValueUsedOutsideLoop", ORE, TheLoop,
- Phi);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Found an unidentified PHI",
+ "value that could not be identified as "
+ "reduction is used outside the loop",
+ "NonReductionValueUsedOutsideLoop", ORE, TheLoop, Phi);
return false;
} // end of PHI handling
@@ -976,16 +985,17 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
// but it's hard to provide meaningful yet generic advice.
// Also, should this be guarded by allowExtraAnalysis() and/or be part
// of the returned info from isFunctionVectorizable()?
- reportVectorizationFailure(
- "Found a non-intrinsic callsite",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Found a non-intrinsic callsite",
"library call cannot be vectorized. "
"Try compiling with -fno-math-errno, -ffast-math, "
"or similar flags",
"CantVectorizeLibcall", ORE, TheLoop, CI);
} else {
- reportVectorizationFailure("Found a non-intrinsic callsite",
- "call instruction cannot be vectorized",
- "CantVectorizeLibcall", ORE, TheLoop, CI);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Found a non-intrinsic callsite",
+ "call instruction cannot be vectorized", "CantVectorizeLibcall", ORE,
+ TheLoop, CI);
}
return false;
}
@@ -998,7 +1008,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
for (unsigned Idx = 0; Idx < CI->arg_size(); ++Idx)
if (isVectorIntrinsicWithScalarOpAtArg(IntrinID, Idx, TTI)) {
if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(Idx)), TheLoop)) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Found unvectorizable intrinsic",
"intrinsic instruction cannot be vectorized",
"CantVectorizeIntrinsic", ORE, TheLoop, CI);
@@ -1031,10 +1042,10 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
(isa<CastInst>(I) &&
!VectorType::isValidElementType(I.getOperand(0)->getType())) ||
isa<ExtractElementInst>(I)) {
- reportVectorizationFailure("Found unvectorizable type",
- "instruction return type cannot be vectorized",
- "CantVectorizeInstructionReturnType", ORE,
- TheLoop, &I);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Found unvectorizable type",
+ "instruction return type cannot be vectorized",
+ "CantVectorizeInstructionReturnType", ORE, TheLoop, &I);
return false;
}
@@ -1042,8 +1053,10 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
if (auto *ST = dyn_cast<StoreInst>(&I)) {
Type *T = ST->getValueOperand()->getType();
if (!VectorType::isValidElementType(T)) {
- reportVectorizationFailure("Store instruction cannot be vectorized",
- "CantVectorizeStore", ORE, TheLoop, ST);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Store instruction cannot be vectorized", "CantVectorizeStore", ORE,
+ TheLoop, ST);
return false;
}
@@ -1054,7 +1067,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
auto *VecTy = FixedVectorType::get(T, /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of stored type");
if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"nontemporal store instruction cannot be vectorized",
"CantVectorizeNontemporalStore", ORE, TheLoop, ST);
return false;
@@ -1068,7 +1082,8 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
auto *VecTy = FixedVectorType::get(I.getType(), /*NumElts=*/2);
assert(VecTy && "did not find vectorized version of load type");
if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"nontemporal load instruction cannot be vectorized",
"CantVectorizeNontemporalLoad", ORE, TheLoop, LD);
return false;
@@ -1097,8 +1112,10 @@ bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
AllowedExit.insert(&I);
return true;
}
- reportVectorizationFailure("Value cannot be used outside the loop",
- "ValueUsedOutsideLoop", ORE, TheLoop, &I);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Value cannot be used outside the loop", "ValueUsedOutsideLoop", ORE,
+ TheLoop, &I);
return false;
}
@@ -1244,7 +1261,8 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
if (!LAI->canVectorizeMemory()) {
if (hasUncountableExitWithSideEffects()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Cannot vectorize unsafe dependencies in uncountable exit loop with "
"side effects",
"CantVectorizeUnsafeDependencyForEELoopWithSideEffects", ORE,
@@ -1256,11 +1274,12 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
}
if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) {
- reportVectorizationFailure("We don't allow storing to uniform addresses",
- "write to a loop invariant address could not "
- "be vectorized",
- "CantVectorizeStoreToLoopInvariantAddress", ORE,
- TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "We don't allow storing to uniform addresses",
+ "write to a loop invariant address could not "
+ "be vectorized",
+ "CantVectorizeStoreToLoopInvariantAddress", ORE, TheLoop);
return false;
}
@@ -1276,7 +1295,8 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
continue;
if (blockNeedsPredication(SI->getParent())) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"We don't allow storing to uniform addresses",
"write of conditional recurring variant value to a loop "
"invariant address could not be vectorized",
@@ -1289,7 +1309,8 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
// to overcomplicate vectorization to support this case.
if (Instruction *Ptr = dyn_cast<Instruction>(SI->getPointerOperand())) {
if (TheLoop->contains(Ptr)) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Invariant address is calculated inside the loop",
"write to a loop invariant address could not "
"be vectorized",
@@ -1332,7 +1353,8 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
bool IsOK = UnhandledStores.empty();
// TODO: we should also validate against InvariantMemSets.
if (!IsOK) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"We don't allow storing to uniform addresses",
"write to a loop invariant address could not "
"be vectorized",
@@ -1502,8 +1524,9 @@ bool LoopVectorizationLegality::blockCanBePredicated(
bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
if (!EnableIfConversion) {
- reportVectorizationFailure("If-conversion is disabled",
- "IfConversionDisabled", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "If-conversion is disabled",
+ "IfConversionDisabled", ORE, TheLoop);
return false;
}
@@ -1593,22 +1616,26 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
// loop.
if (isa<SwitchInst>(BB->getTerminator())) {
if (TheLoop->isLoopExiting(BB)) {
- reportVectorizationFailure("Loop contains an unsupported switch",
- "LoopContainsUnsupportedSwitch", ORE,
- TheLoop, BB->getTerminator());
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Loop contains an unsupported switch",
+ "LoopContainsUnsupportedSwitch", ORE, TheLoop, BB->getTerminator());
return false;
}
} else if (!isa<UncondBrInst, CondBrInst>(BB->getTerminator())) {
- reportVectorizationFailure("Loop contains an unsupported terminator",
- "LoopContainsUnsupportedTerminator", ORE,
- TheLoop, BB->getTerminator());
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Loop contains an unsupported terminator",
+ "LoopContainsUnsupportedTerminator", ORE, TheLoop,
+ BB->getTerminator());
return false;
}
// We must be able to predicate all blocks that need to be predicated.
if (blockNeedsPredication(BB) &&
!blockCanBePredicated(BB, SafePointers, ConditionallyExecutedOps)) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Control flow cannot be substituted for a select", "NoCFGForSelect",
ORE, TheLoop, BB->getTerminator());
return false;
@@ -1638,9 +1665,11 @@ bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
// We must have a loop in canonical form. Loops with indirectbr in them cannot
// be canonicalized.
if (!Lp->getLoopPreheader()) {
- reportVectorizationFailure("Loop doesn't have a legal pre-header",
- "loop control flow is not understood by vectorizer",
- "CFGNotUnderstood", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Loop doesn't have a legal pre-header",
+ "loop control flow is not understood by vectorizer", "CFGNotUnderstood",
+ ORE, TheLoop);
if (DoExtraAnalysis)
Result = false;
else
@@ -1649,9 +1678,11 @@ bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
// We must have a single backedge.
if (Lp->getNumBackEdges() != 1) {
- reportVectorizationFailure("The loop must have a single backedge",
- "loop control flow is not understood by vectorizer",
- "CFGNotUnderstood", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "The loop must have a single backedge",
+ "loop control flow is not understood by vectorizer", "CFGNotUnderstood",
+ ORE, TheLoop);
if (DoExtraAnalysis)
Result = false;
else
@@ -1661,7 +1692,8 @@ bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
// The latch must be terminated by a branch.
BasicBlock *Latch = Lp->getLoopLatch();
if (Latch && !isa<UncondBrInst, CondBrInst>(Latch->getTerminator())) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"The loop latch terminator is not a UncondBrInst/CondBrInst",
"loop control flow is not understood by vectorizer", "CFGNotUnderstood",
ORE, TheLoop);
@@ -1703,14 +1735,15 @@ bool LoopVectorizationLegality::canVectorizeLoopNestCFG(
bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
BasicBlock *LatchBB = TheLoop->getLoopLatch();
if (!LatchBB) {
- reportVectorizationFailure("Loop does not have a latch",
- "Cannot vectorize early exit loop",
- "NoLatchEarlyExit", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Loop does not have a latch",
+ "Cannot vectorize early exit loop", "NoLatchEarlyExit", ORE, TheLoop);
return false;
}
if (Reductions.size() || FixedOrderRecurrences.size()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Found reductions or recurrences in early-exit loop",
"Cannot vectorize early exit loop with reductions or recurrences",
"RecurrencesInEarlyExitLoop", ORE, TheLoop);
@@ -1728,7 +1761,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
if (isa<SCEVCouldNotCompute>(EC)) {
if (size(successors(BB)) != 2) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Early exiting block does not have exactly two successors",
"Incorrect number of successors from early exiting block",
"EarlyExitTooManySuccessors", ORE, TheLoop);
@@ -1753,7 +1787,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// The latch block must have a countable exit.
if (isa<SCEVCouldNotCompute>(
PSE.getSE()->getPredicatedExitCount(TheLoop, LatchBB, &Predicates))) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Cannot determine exact exit count for latch block",
"Cannot vectorize early exit loop",
"UnknownLatchExitCountEarlyExitLoop", ORE, TheLoop);
@@ -1788,7 +1823,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
// We don't support complex writes to memory.
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Complex writes to memory unsupported in early exit loops",
"Cannot vectorize early exit loop with complex writes to memory",
"WritesInEarlyExitLoop", ORE, TheLoop);
@@ -1796,10 +1832,11 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
if (!IsSafeOperation(&I)) {
- reportVectorizationFailure("Early exit loop contains operations that "
- "cannot be speculatively executed",
- "UnsafeOperationsEarlyExitLoop", ORE,
- TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Early exit loop contains operations that "
+ "cannot be speculatively executed",
+ "UnsafeOperationsEarlyExitLoop", ORE, TheLoop);
return false;
}
}
@@ -1811,8 +1848,9 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
Predicates.clear();
if (!isReadOnlyLoop(TheLoop, PSE.getSE(), DT, AC, NonDerefLoads,
&Predicates)) {
- reportVectorizationFailure(
- "Loop may fault", "Cannot vectorize non-read-only early exit loop",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Loop may fault",
+ "Cannot vectorize non-read-only early exit loop",
"NonReadOnlyEarlyExitLoop", ORE, TheLoop);
return false;
}
@@ -1829,7 +1867,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// Only support unit-stride access for now.
int Stride = isConsecutivePtr(LI->getType(), LI->getPointerOperand());
if (Stride != 1) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Loop contains potentially faulting strided load",
"Cannot vectorize early exit loop with "
"strided fault-only-first load",
@@ -1870,7 +1909,8 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
if (!match(Br->getCondition(),
m_OneUse(m_ICmp(m_OneUse(m_Instruction(L, m_Load(m_Value(Ptr)))),
m_Value(R))))) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Early exit loop with store but no supported condition load",
"NoConditionLoadForEarlyExitLoop", ORE, TheLoop);
return false;
@@ -1878,7 +1918,8 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
// FIXME: Don't rely on operand ordering for the comparison.
if (!TheLoop->isLoopInvariant(R)) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Early exit loop with store but no supported condition load",
"NoConditionLoadForEarlyExitLoop", ORE, TheLoop);
return false;
@@ -1888,7 +1929,8 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
// address calculation that we can rotate to the next vector iteration.
const auto *AR = dyn_cast<SCEVAddRecExpr>(PSE.getSE()->getSCEV(Ptr));
if (!AR || AR->getLoop() != TheLoop || !AR->isAffine()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Uncountable exit condition depends on load with an address that is "
"not an add recurrence in the loop",
"EarlyExitLoadInvariantAddress", ORE, TheLoop);
@@ -1901,7 +1943,8 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
// We need to know that load will be executed before we can hoist a
// copy out to run just before the first iteration.
if (!SafetyInfo.isGuaranteedToExecute(*Load, DT, TheLoop)) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Load for uncountable exit not guaranteed to execute",
"ConditionalUncountableExitLoad", ORE, TheLoop);
return false;
@@ -1922,7 +1965,8 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
continue;
}
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Cannot determine whether critical uncountable exit load address "
"does not alias with a memory write",
"CantVectorizeAliasWithCriticalUncountableExitLoad", ORE, TheLoop);
@@ -1961,8 +2005,9 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
assert(UseVPlanNativePath && "VPlan-native path is not enabled.");
if (!canVectorizeOuterLoop()) {
- reportVectorizationFailure("Unsupported outer loop",
- "UnsupportedOuterLoop", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Unsupported outer loop",
+ "UnsupportedOuterLoop", ORE, TheLoop);
// TODO: Implement DoExtraAnalysis when subsequent legal checks support
// outer loops.
return false;
@@ -1994,8 +2039,10 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
if (TheLoop->getExitingBlock()) {
- reportVectorizationFailure("Cannot vectorize uncountable loop",
- "UnsupportedUncountableLoop", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
+ "Cannot vectorize uncountable loop", "UnsupportedUncountableLoop",
+ ORE, TheLoop);
if (DoExtraAnalysis)
Result = false;
else
@@ -2023,7 +2070,8 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
// Bail out for ReadWrite loops with uncountable exits for now.
if (UncountableExitType == UncountableExitTrait::ReadWrite) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Writes to memory unsupported in early exit loops",
"Cannot vectorize early exit loop with writes to memory",
"WritesInEarlyExitLoop", ORE, TheLoop);
@@ -2045,7 +2093,8 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
if (PSE.getPredicate().getComplexity() > SCEVThreshold) {
LLVM_DEBUG(dbgs() << "LV: Vectorization not profitable "
"due to SCEVThreshold");
- reportVectorizationFailure("Too many SCEV checks needed",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Too many SCEV checks needed",
"Too many SCEV assumptions need to be made and checked at runtime",
"TooManySCEVRunTimeChecks", ORE, TheLoop);
if (DoExtraAnalysis)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 0e847f4767a8b..7493e5bb10b2e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "LoopVectorizationPlanner.h"
+#include "LoopVectorizationUtils.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
@@ -247,8 +248,10 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
return false;
if (Hints->isScalableVectorizationDisabled()) {
- reportVectorizationInfo("Scalable vectorization is explicitly disabled",
- "ScalableVectorizationDisabled", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints->vectorizeAnalysisPassName(),
+ "Scalable vectorization is explicitly disabled",
+ "ScalableVectorizationDisabled", ORE, TheLoop);
return false;
}
@@ -267,7 +270,8 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
if (!all_of(Legal->getReductionVars(), [&](const auto &Reduction) -> bool {
return TTI.isLegalToVectorizeReduction(Reduction.second, MaxScalableVF);
})) {
- reportVectorizationInfo(
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints->vectorizeAnalysisPassName(),
"Scalable vectorization not supported for the reduction "
"operations found in this loop.",
"ScalableVFUnfeasible", ORE, TheLoop);
@@ -279,16 +283,20 @@ bool VFSelectionContext::isScalableVectorizationAllowed() {
if (any_of(ElementTypesInLoop, [&](Type *Ty) {
return !Ty->isVoidTy() && !TTI.isElementTypeLegalForScalableVector(Ty);
})) {
- reportVectorizationInfo("Scalable vectorization is not supported "
- "for all element types found in this loop.",
- "ScalableVFUnfeasible", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints->vectorizeAnalysisPassName(),
+ "Scalable vectorization is not supported "
+ "for all element types found in this loop.",
+ "ScalableVFUnfeasible", ORE, TheLoop);
return false;
}
if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(F, TTI)) {
- reportVectorizationInfo("The target does not provide maximum vscale value "
- "for safe distance analysis.",
- "ScalableVFUnfeasible", ORE, TheLoop);
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints->vectorizeAnalysisPassName(),
+ "The target does not provide maximum vscale value "
+ "for safe distance analysis.",
+ "ScalableVFUnfeasible", ORE, TheLoop);
return false;
}
@@ -311,7 +319,8 @@ VFSelectionContext::getMaxLegalScalableVF(unsigned MaxSafeElements) {
MaxScalableVF = ElementCount::getScalable(MaxSafeElements / *MaxVScale);
if (!MaxScalableVF)
- reportVectorizationInfo(
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints->vectorizeAnalysisPassName(),
"Max legal vector width too small, scalable vectorization "
"unfeasible.",
"ScalableVFUnfeasible", ORE, TheLoop);
@@ -531,7 +540,8 @@ bool VFSelectionContext::runtimeChecksRequired() {
Loop *L = const_cast<Loop *>(TheLoop);
if (Legal->getRuntimePointerChecking()->Need) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Runtime ptr check is required with -Os/-Oz",
"runtime pointer checks needed. Enable vectorization of this "
"loop with '#pragma clang loop vectorize(enable)' when "
@@ -541,7 +551,8 @@ bool VFSelectionContext::runtimeChecksRequired() {
}
if (!PSE.getPredicate().isAlwaysTrue()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Runtime SCEV check is required with -Os/-Oz",
"runtime SCEV checks needed. Enable vectorization of this "
"loop with '#pragma clang loop vectorize(enable)' when "
@@ -552,7 +563,8 @@ bool VFSelectionContext::runtimeChecksRequired() {
// FIXME: Avoid specializing for stride==1 instead of bailing out.
if (!Legal->getLAI()->getSymbolicStrides().empty()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Runtime stride check for small trip count",
"runtime stride == 1 checks needed. Enable vectorization of "
"this loop without such check by compiling with -Os/-Oz",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b07a17f2d8baa..e2cecd2be6aaf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -24,6 +24,7 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
+#include "LoopVectorizationUtils.h"
#include "VPlan.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -58,15 +59,6 @@ extern cl::opt<bool> PreferInLoopReductions;
std::optional<unsigned> getMaxVScale(const Function &F,
const TargetTransformInfo &TTI);
-/// Reports an informative message: print \p Msg for debugging purposes as well
-/// as an optimization remark. Uses either \p I as location of the remark, or
-/// otherwise \p TheLoop. If \p DL is passed, use it as debug location for the
-/// remark.
-void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
- OptimizationRemarkEmitter *ORE,
- const Loop *TheLoop, Instruction *I = nullptr,
- DebugLoc DL = {});
-
/// VPlan-based builder utility analogous to IRBuilder.
class VPBuilder {
VPBasicBlock *BB = nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.cpp
new file mode 100644
index 0000000000000..e52e386d16097
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.cpp
@@ -0,0 +1,132 @@
+//===----------- LoopVectorizationUtils.h - Utilities for LoopVectorize
+//-----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements utilities functions that are used by the LoopVectorize
+/// and its related files. Those utilities don't depend on any state.
+//===----------------------------------------------------------------------===//
+
+#include "LoopVectorizationUtils.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "loop-vectorize"
+
+using namespace llvm;
+using namespace SCEVPatternMatch;
+
+/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
+/// is passed, the message relates to that particular instruction.
+#ifndef NDEBUG
+void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg,
+ Instruction *I) {
+ dbgs() << "LV: " << Prefix << DebugMsg;
+ if (I != nullptr)
+ dbgs() << " " << *I;
+ else
+ dbgs() << '.';
+ dbgs() << '\n';
+}
+#endif
+
+/// Create an analysis remark that explains why vectorization failed
+///
+/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
+/// RemarkName is the identifier for the remark. If \p I is passed it is an
+/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
+/// the location of the remark. If \p DL is passed, use it as debug location for
+/// the remark. \return the remark object that can be streamed to.
+OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
+ StringRef RemarkName,
+ const Loop *TheLoop, Instruction *I,
+ DebugLoc DL = {}) {
+ BasicBlock *CodeRegion = I ? I->getParent() : TheLoop->getHeader();
+ // If debug location is attached to the instruction, use it. Otherwise if DL
+ // was not provided, use the loop's.
+ if (I && I->getDebugLoc())
+ DL = I->getDebugLoc();
+ else if (!DL)
+ DL = TheLoop->getStartLoc();
+
+ return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
+}
+
+void LoopVectorizationUtils::reportVectorizationFailure(
+ const char *PassName, const StringRef DebugMsg, const StringRef OREMsg,
+ const StringRef ORETag, OptimizationRemarkEmitter *ORE, const Loop *TheLoop,
+ Instruction *I) {
+ LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I));
+ ORE->emit(createLVAnalysis(PassName, ORETag, TheLoop, I)
+ << "loop not vectorized: " << OREMsg);
+}
+
+void LoopVectorizationUtils::reportVectorizationInfo(
+ const char *PassName, const StringRef Msg, const StringRef ORETag,
+ OptimizationRemarkEmitter *ORE, const Loop *TheLoop, Instruction *I,
+ DebugLoc DL) {
+ LLVM_DEBUG(debugVectorizationMessage("", Msg, I));
+ ORE->emit(createLVAnalysis(PassName, ORETag, TheLoop, I, DL) << Msg);
+}
+
+void LoopVectorizationUtils::reportVectorization(const char *PassName,
+ OptimizationRemarkEmitter *ORE,
+ Loop *TheLoop,
+ ElementCount VFWidth,
+ unsigned IC) {
+ LLVM_DEBUG(debugVectorizationMessage(
+ "Vectorizing: ", TheLoop->isInnermost() ? "innermost loop" : "outer loop",
+ nullptr));
+ StringRef LoopType = TheLoop->isInnermost() ? "" : "outer ";
+ ORE->emit([&]() {
+ return OptimizationRemark(PassName, "Vectorized", TheLoop->getStartLoc(),
+ TheLoop->getHeader())
+ << "vectorized " << LoopType << "loop (vectorization width: "
+ << ore::NV("VectorizationFactor", VFWidth)
+ << ", interleaved count: " << ore::NV("InterleaveCount", IC) << ")";
+ });
+}
+
+ElementCount
+LoopVectorizationUtils::getSmallConstantTripCount(ScalarEvolution *SE,
+ const Loop *L) {
+ if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
+ return ElementCount::getFixed(ExpectedTC);
+
+ const SCEV *BTC = SE->getBackedgeTakenCount(L);
+ if (isa<SCEVCouldNotCompute>(BTC))
+ return ElementCount::getFixed(0);
+
+ const SCEV *ExitCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
+ if (isa<SCEVVScale>(ExitCount))
+ return ElementCount::getScalable(1);
+
+ const APInt *Scale;
+ if (match(ExitCount, m_scev_Mul(m_scev_APInt(Scale), m_SCEVVScale())))
+ if (cast<SCEVMulExpr>(ExitCount)->hasNoUnsignedWrap())
+ if (Scale->getActiveBits() <= 32)
+ return ElementCount::getScalable(Scale->getZExtValue());
+
+ return ElementCount::getFixed(0);
+}
+
+unsigned
+LoopVectorizationUtils::getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
+ const Loop *L) {
+ const SCEV *BTC = PSE.getBackedgeTakenCount();
+ if (isa<SCEVCouldNotCompute>(BTC))
+ return 0;
+ ScalarEvolution *SE = PSE.getSE();
+ const SCEV *TripCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
+ ConstantRange TCRange = SE->getUnsignedRange(TripCount);
+ APInt MaxTCFromRange = TCRange.getUnsignedMax();
+ if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
+ return MaxTCFromRange.getZExtValue();
+ return 0;
+}
\ No newline at end of file
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.h
new file mode 100644
index 0000000000000..456322ab2fc33
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationUtils.h
@@ -0,0 +1,103 @@
+//===----------- VectorizeUtils.h - Utilities for LoopVectorize -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides declaration for utilities functions that are used by the
+/// LoopVectorize and its related files. Those utilities don't depend on any
+/// state.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONUTILS_H
+#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONUTILS_H
+
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+namespace LoopVectorizationUtils {
+
+// Loop vectorization hints how the epilogue/tail loop should be
+// lowered.
+enum EpilogueLowering {
+
+ // The default: allowing epilogues.
+ EpilogueAllowed,
+
+ // Vectorization with OptForSize: don't allow epilogues.
+ EpilogueNotAllowedOptSize,
+
+ // A special case of vectorisation with OptForSize: loops with a very small
+ // trip count are considered for vectorization under OptForSize, thereby
+ // making sure the cost of their loop body is dominant, free of runtime
+ // guards and scalar iteration overheads.
+ EpilogueNotAllowedLowTripLoop,
+
+ // Loop hint indicating an epilogue is undesired, apply tail folding.
+ EpilogueNotNeededFoldTail,
+
+ // Tail-folded vector epilogue requested; the scalar tail will be folded into
+ // the vectorized epilogue loop if possible, otherwise fall back to an
+ // epilogue. This status is specifically for the CM instance of the
+ // tail-folded epilogue.
+ EpilogueNotNeededFoldEpilogueTail,
+
+ // Directive indicating we must either fold the epilogue/tail or not vectorize
+ EpilogueNotAllowedFoldTail
+};
+
+/// Reports a vectorization failure: print \p DebugMsg for debugging
+/// purposes along with the corresponding optimization remark \p RemarkName.
+/// If \p I is passed, it is an instruction that prevents vectorization.
+/// Otherwise, the loop \p TheLoop is used for the location of the remark.
+void reportVectorizationFailure(const char *PassName, const StringRef DebugMsg,
+ const StringRef OREMsg, const StringRef ORETag,
+ OptimizationRemarkEmitter *ORE,
+ const Loop *TheLoop, Instruction *I = nullptr);
+
+/// Same as above, but the debug message and optimization remark are identical
+inline void reportVectorizationFailure(const char *PassName,
+ const StringRef DebugMsg,
+ const StringRef ORETag,
+ OptimizationRemarkEmitter *ORE,
+ const Loop *TheLoop,
+ Instruction *I = nullptr) {
+ reportVectorizationFailure(PassName, DebugMsg, DebugMsg, ORETag, ORE, TheLoop,
+ I);
+}
+
+/// Reports an informative message: print \p Msg for debugging purposes as well
+/// as an optimization remark. Uses either \p I as location of the remark, or
+/// otherwise \p TheLoop. If \p DL is passed, use it as debug location for the
+/// remark.
+void reportVectorizationInfo(const char *PassName, const StringRef Msg,
+ const StringRef ORETag,
+ OptimizationRemarkEmitter *ORE,
+ const Loop *TheLoop, Instruction *I = nullptr,
+ DebugLoc DL = {});
+
+/// Report successful vectorization of the loop. In case an outer loop is
+/// vectorized, prepend "outer" to the vectorization remark.
+void reportVectorization(const char *PassName, OptimizationRemarkEmitter *ORE,
+ Loop *TheLoop, ElementCount VFWidth, unsigned IC);
+
+/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
+/// ElementCount to include loops whose trip count is a function of vscale.
+ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L);
+
+/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
+/// zero from the range. Only valid when not folding the tail, as the minimum
+/// iteration count check guards against a zero trip count. Returns 0 if
+/// unknown.
+unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
+ const Loop *L);
+} // namespace LoopVectorizationUtils
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONUTILS_H
\ No newline at end of file
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3d33c46075b05..a6aa9c2c44510 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -55,6 +55,7 @@
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
+#include "LoopVectorizationUtils.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
@@ -388,48 +389,6 @@ static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
}
-/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
-/// ElementCount to include loops whose trip count is a function of vscale.
-static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
- const Loop *L) {
- if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
- return ElementCount::getFixed(ExpectedTC);
-
- const SCEV *BTC = SE->getBackedgeTakenCount(L);
- if (isa<SCEVCouldNotCompute>(BTC))
- return ElementCount::getFixed(0);
-
- const SCEV *ExitCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
- if (isa<SCEVVScale>(ExitCount))
- return ElementCount::getScalable(1);
-
- const APInt *Scale;
- if (match(ExitCount, m_scev_Mul(m_scev_APInt(Scale), m_SCEVVScale())))
- if (cast<SCEVMulExpr>(ExitCount)->hasNoUnsignedWrap())
- if (Scale->getActiveBits() <= 32)
- return ElementCount::getScalable(Scale->getZExtValue());
-
- return ElementCount::getFixed(0);
-}
-
-/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
-/// zero from the range. Only valid when not folding the tail, as the minimum
-/// iteration count check guards against a zero trip count. Returns 0 if
-/// unknown.
-static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
- Loop *L) {
- const SCEV *BTC = PSE.getBackedgeTakenCount();
- if (isa<SCEVCouldNotCompute>(BTC))
- return 0;
- ScalarEvolution *SE = PSE.getSE();
- const SCEV *TripCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
- ConstantRange TCRange = SE->getUnsignedRange(TripCount);
- APInt MaxTCFromRange = TCRange.getUnsignedMax();
- if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
- return MaxTCFromRange.getZExtValue();
- return 0;
-}
-
/// Returns "best known" trip count, which is either a valid positive trip count
/// or std::nullopt when an estimate cannot be made (including when the trip
/// count would overflow), for the specified loop \p L as defined by the
@@ -445,7 +404,8 @@ getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L,
bool CanUseConstantMax = true,
bool CanExcludeZeroTrips = false) {
// Check if exact trip count is known.
- if (auto ExpectedTC = getSmallConstantTripCount(PSE.getSE(), L))
+ if (auto ExpectedTC =
+ LoopVectorizationUtils::getSmallConstantTripCount(PSE.getSE(), L))
return ExpectedTC;
// Check if there is an expected trip count available from profile data.
@@ -464,7 +424,8 @@ getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L,
// only safe when not folding the tail, as the minimum iteration count check
// prevents entering the vector loop with a zero trip count.
if (CanUseConstantMax && CanExcludeZeroTrips)
- if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
+ if (unsigned RefinedTC =
+ LoopVectorizationUtils::getMaxTCFromNonZeroRange(PSE, L))
return ElementCount::getFixed(RefinedTC);
return std::nullopt;
@@ -700,42 +661,6 @@ static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
return I->getDebugLoc();
}
-/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
-/// is passed, the message relates to that particular instruction.
-#ifndef NDEBUG
-static void debugVectorizationMessage(const StringRef Prefix,
- const StringRef DebugMsg,
- Instruction *I) {
- dbgs() << "LV: " << Prefix << DebugMsg;
- if (I != nullptr)
- dbgs() << " " << *I;
- else
- dbgs() << '.';
- dbgs() << '\n';
-}
-#endif
-
-/// Create an analysis remark that explains why vectorization failed
-///
-/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
-/// RemarkName is the identifier for the remark. If \p I is passed it is an
-/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
-/// the location of the remark. If \p DL is passed, use it as debug location for
-/// the remark. \return the remark object that can be streamed to.
-static OptimizationRemarkAnalysis
-createLVAnalysis(const char *PassName, StringRef RemarkName,
- const Loop *TheLoop, Instruction *I, DebugLoc DL = {}) {
- BasicBlock *CodeRegion = I ? I->getParent() : TheLoop->getHeader();
- // If debug location is attached to the instruction, use it. Otherwise if DL
- // was not provided, use the loop's.
- if (I && I->getDebugLoc())
- DL = I->getDebugLoc();
- else if (!DL)
- DL = TheLoop->getStartLoc();
-
- return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
-}
-
namespace llvm {
/// Return the runtime value for VF.
@@ -743,71 +668,10 @@ Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
return B.CreateElementCount(Ty, VF);
}
-void reportVectorizationFailure(const StringRef DebugMsg,
- const StringRef OREMsg, const StringRef ORETag,
- OptimizationRemarkEmitter *ORE, Loop *TheLoop,
- Instruction *I) {
- LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I));
- LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
- ORE->emit(
- createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
- << "loop not vectorized: " << OREMsg);
-}
-
-void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
- OptimizationRemarkEmitter *ORE,
- const Loop *TheLoop, Instruction *I, DebugLoc DL) {
- LLVM_DEBUG(debugVectorizationMessage("", Msg, I));
- LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
- ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop,
- I, DL)
- << Msg);
-}
-
-/// Report successful vectorization of the loop. In case an outer loop is
-/// vectorized, prepend "outer" to the vectorization remark.
-static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop,
- VectorizationFactor VF, unsigned IC) {
- LLVM_DEBUG(debugVectorizationMessage(
- "Vectorizing: ", TheLoop->isInnermost() ? "innermost loop" : "outer loop",
- nullptr));
- StringRef LoopType = TheLoop->isInnermost() ? "" : "outer ";
- ORE->emit([&]() {
- return OptimizationRemark(LV_NAME, "Vectorized", TheLoop->getStartLoc(),
- TheLoop->getHeader())
- << "vectorized " << LoopType << "loop (vectorization width: "
- << ore::NV("VectorizationFactor", VF.Width)
- << ", interleaved count: " << ore::NV("InterleaveCount", IC) << ")";
- });
-}
-
} // end namespace llvm
namespace llvm {
-// Loop vectorization cost-model hints how the epilogue/tail loop should be
-// lowered.
-enum EpilogueLowering {
-
- // The default: allowing epilogues.
- CM_EpilogueAllowed,
-
- // Vectorization with OptForSize: don't allow epilogues.
- CM_EpilogueNotAllowedOptSize,
-
- // A special case of vectorisation with OptForSize: loops with a very small
- // trip count are considered for vectorization under OptForSize, thereby
- // making sure the cost of their loop body is dominant, free of runtime
- // guards and scalar iteration overheads.
- CM_EpilogueNotAllowedLowTripLoop,
-
- // Loop hint indicating an epilogue is undesired, apply tail folding.
- CM_EpilogueNotNeededFoldTail,
-
- // Directive indicating we must either fold the epilogue/tail or not vectorize
- CM_EpilogueNotAllowedFoldTail
-};
-
/// LoopVectorizationCostModel - estimates the expected speedups due to
/// vectorization.
/// In many cases vectorization is not profitable. This can happen because of
@@ -819,14 +683,17 @@ class LoopVectorizationCostModel {
friend class LoopVectorizationPlanner;
public:
- LoopVectorizationCostModel(
- EpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE,
- LoopInfo *LI, LoopVectorizationLegality *Legal,
- const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI,
- DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE,
- std::function<BlockFrequencyInfo &()> GetBFI, const Function *F,
- const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI,
- VFSelectionContext &Config)
+ LoopVectorizationCostModel(LoopVectorizationUtils::EpilogueLowering SEL,
+ Loop *L, PredicatedScalarEvolution &PSE,
+ LoopInfo *LI, LoopVectorizationLegality *Legal,
+ const TargetTransformInfo &TTI,
+ const TargetLibraryInfo *TLI, DemandedBits *DB,
+ AssumptionCache *AC,
+ OptimizationRemarkEmitter *ORE,
+ std::function<BlockFrequencyInfo &()> GetBFI,
+ const Function *F, const LoopVectorizeHints *Hints,
+ InterleavedAccessInfo &IAI,
+ VFSelectionContext &Config)
: Config(Config), EpilogueLoweringStatus(SEL), TheLoop(L), PSE(PSE),
LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE),
GetBFI(GetBFI), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {}
@@ -1173,13 +1040,15 @@ class LoopVectorizationCostModel {
/// Returns true if an epilogue is allowed (e.g., not prevented by
/// optsize or a loop hint annotation).
bool isEpilogueAllowed() const {
- return EpilogueLoweringStatus == CM_EpilogueAllowed;
+ return EpilogueLoweringStatus == LoopVectorizationUtils::EpilogueAllowed;
}
/// Returns true if tail-folding is preferred over an epilogue.
bool preferTailFoldedLoop() const {
- return EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail ||
- EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail;
+ return EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotNeededFoldTail ||
+ EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotAllowedFoldTail;
}
/// Returns the TailFoldingStyle that is best for the current loop.
@@ -1213,8 +1082,9 @@ class LoopVectorizationCostModel {
return;
// If for some reason EVL mode is unsupported, fallback to an epilogue
// if it's allowed, or DataWithoutLaneMask otherwise.
- if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
- EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
+ if (EpilogueLoweringStatus == LoopVectorizationUtils::EpilogueAllowed ||
+ EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotNeededFoldTail)
ChosenTailFoldingStyle = TailFoldingStyle::None;
else
ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
@@ -1373,7 +1243,8 @@ class LoopVectorizationCostModel {
/// or as a peel-loop to handle gaps in interleave-groups.
/// Under optsize and when the trip count is very small we don't allow any
/// iterations to execute in the scalar loop.
- EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
+ LoopVectorizationUtils::EpilogueLowering EpilogueLoweringStatus =
+ LoopVectorizationUtils::EpilogueAllowed;
/// Control finally chosen tail folding style.
TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
@@ -2977,7 +2848,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
// TODO: It may be useful to do since it's still likely to be dynamically
// uniform if the target can skip.
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Not inserting runtime ptr check for divergent target",
"runtime pointer checks needed. Not enabled for divergent target",
"CantVersionLoopWithDivergentTarget", ORE, TheLoop);
@@ -2985,15 +2857,18 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
ScalarEvolution *SE = PSE.getSE();
- ElementCount TC = getSmallConstantTripCount(SE, TheLoop);
+ ElementCount TC =
+ LoopVectorizationUtils::getSmallConstantTripCount(SE, TheLoop);
unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
- if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
- MaxTC = getMaxTCFromNonZeroRange(PSE, TheLoop);
+ if (!MaxTC &&
+ EpilogueLoweringStatus == LoopVectorizationUtils::EpilogueAllowed)
+ MaxTC = LoopVectorizationUtils::getMaxTCFromNonZeroRange(PSE, TheLoop);
LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
if (TC != ElementCount::getFixed(MaxTC))
LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
if (TC.isScalar()) {
- reportVectorizationFailure("Single iteration (non) loop",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Single iteration (non) loop",
"loop trip count is one, irrelevant for vectorization",
"SingleIterationLoop", ORE, TheLoop);
return FixedScalableVFPair::getNone();
@@ -3008,8 +2883,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
Legal->getWidestInductionType()->getScalarSizeInBits() &&
SE->isKnownPredicate(CmpInst::ICMP_EQ, BTC,
SE->getMinusOne(BTC->getType()))) {
- reportVectorizationFailure(
- "Trip count computation wrapped",
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(), "Trip count computation wrapped",
"backedge-taken count is -1, loop trip count wrapped to 0",
"TripCountWrapped", ORE, TheLoop);
return FixedScalableVFPair::getNone();
@@ -3022,20 +2897,21 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
switch (EpilogueLoweringStatus) {
- case CM_EpilogueAllowed:
+ case LoopVectorizationUtils::EpilogueAllowed:
return Config.computeFeasibleMaxVF(MaxTC, UserVF, UserIC, false,
requiresScalarEpilogue(true));
- case CM_EpilogueNotAllowedFoldTail:
+ case LoopVectorizationUtils::EpilogueNotAllowedFoldTail:
[[fallthrough]];
- case CM_EpilogueNotNeededFoldTail:
+ case LoopVectorizationUtils::EpilogueNotNeededFoldTail:
LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
<< "LV: Not allowing epilogue, creating tail-folded "
<< "vector loop.\n");
break;
- case CM_EpilogueNotAllowedLowTripLoop:
+ case LoopVectorizationUtils::EpilogueNotAllowedLowTripLoop:
// fallthrough as a special case of OptForSize
- case CM_EpilogueNotAllowedOptSize:
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
+ case LoopVectorizationUtils::EpilogueNotAllowedOptSize:
+ if (EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotAllowedOptSize)
LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
else
LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
@@ -3117,7 +2993,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If we have a low-trip-count, and the fixed-width VF is known to divide
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
+ if (EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotAllowedLowTripLoop &&
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
@@ -3126,7 +3003,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
}
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"The trip count is below the minial threshold value.",
"loop trip count is too low, avoiding vectorization", "LowTripCount",
ORE, TheLoop);
@@ -3158,26 +3036,30 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If there was a tail-folding hint/switch, but we can't fold the tail by
// masking, fallback to a vectorization with an epilogue.
- if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
+ if (EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotNeededFoldTail) {
LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
"epilogue instead.\n");
- EpilogueLoweringStatus = CM_EpilogueAllowed;
+ EpilogueLoweringStatus = LoopVectorizationUtils::EpilogueAllowed;
return MaxFactors;
}
- if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
+ if (EpilogueLoweringStatus ==
+ LoopVectorizationUtils::EpilogueNotAllowedFoldTail) {
LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
return FixedScalableVFPair::getNone();
}
if (TC.isZero()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"unable to calculate the loop count due to complex control flow",
"UnknownLoopCountComplexCFG", ORE, TheLoop);
return FixedScalableVFPair::getNone();
}
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints->vectorizeAnalysisPassName(),
"Cannot optimize for size and vectorize at the same time.",
"cannot optimize for size and vectorize at the same time. "
"Enable vectorization of this loop with '#pragma clang loop "
@@ -3382,8 +3264,9 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
OS << " call to " << Name;
} else
OS << " " << Instruction::getOpcodeName(Opcode);
- reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
- R->getDebugLoc());
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints.vectorizeAnalysisPassName(), OutString, "InvalidCost", ORE,
+ OrigLoop, nullptr, R->getDebugLoc());
Tail = Tail.drop_front(Subset.size());
Subset = {};
} else
@@ -3912,7 +3795,8 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
unsigned InterleaveCountLB = bit_floor(std::max(
1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
- if (getSmallConstantTripCount(PSE.getSE(), OrigLoop).isNonZero()) {
+ if (LoopVectorizationUtils::getSmallConstantTripCount(PSE.getSE(), OrigLoop)
+ .isNonZero()) {
// If the best known trip count is exact, we select between two
// prospective ICs, where
//
@@ -5824,7 +5708,8 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
} else if (UserVF.isScalable() && !Config.supportsScalableVectors()) {
LLVM_DEBUG(dbgs() << "LV: Not vectorizing. Scalable VF requested, but "
<< "not supported by the target.\n");
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints.vectorizeAnalysisPassName(),
"Scalable vectorization requested but not supported by the target",
"the scalable user-specified vectorization width for outer-loop "
"vectorization cannot be used because the target does not support "
@@ -5885,7 +5770,8 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
if (UserVF) {
if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
- reportVectorizationInfo(
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints.vectorizeAnalysisPassName(),
"UserVF ignored because it may be larger than the maximal safe VF",
"InvalidUserVF", ORE, OrigLoop);
} else {
@@ -5914,8 +5800,10 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
}
}
VPlans.clear();
- reportVectorizationInfo("UserVF ignored because of invalid costs.",
- "InvalidCost", ORE, OrigLoop);
+ LoopVectorizationUtils::reportVectorizationInfo(
+ Hints.vectorizeAnalysisPassName(),
+ "UserVF ignored because of invalid costs.", "InvalidCost", ORE,
+ OrigLoop);
}
}
@@ -5999,7 +5887,8 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
// simplified away.
// TODO: Remove this code after stepping away from the legacy cost model and
// adding code to simplify VPlans before calculating their costs.
- auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+ auto TC = LoopVectorizationUtils::getSmallConstantTripCount(PSE.getSE(),
+ OrigLoop);
if (TC == VF && !CM.foldTailByMasking())
addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
CostCtx.SkipCostComputation);
@@ -7517,7 +7406,7 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) tail-folding compiler options, 3) loop
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
-static EpilogueLowering
+static LoopVectorizationUtils::EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL,
@@ -7526,34 +7415,34 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
// don't look at hints or options, and don't request an epilogue.
if (F->hasOptSize() ||
(OptForSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled))
- return CM_EpilogueNotAllowedOptSize;
+ return LoopVectorizationUtils::EpilogueNotAllowedOptSize;
// 2) If set, obey the directives
if (TailFoldingPolicy.getNumOccurrences()) {
switch (TailFoldingPolicy) {
case TailFoldingPolicyTy::None:
- return CM_EpilogueAllowed;
+ return LoopVectorizationUtils::EpilogueAllowed;
case TailFoldingPolicyTy::PreferFoldTail:
- return CM_EpilogueNotNeededFoldTail;
+ return LoopVectorizationUtils::EpilogueNotNeededFoldTail;
case TailFoldingPolicyTy::MustFoldTail:
- return CM_EpilogueNotAllowedFoldTail;
+ return LoopVectorizationUtils::EpilogueNotAllowedFoldTail;
};
}
// 3) If set, obey the hints
switch (Hints.getPredicate()) {
case LoopVectorizeHints::FK_Enabled:
- return CM_EpilogueNotNeededFoldTail;
+ return LoopVectorizationUtils::EpilogueNotNeededFoldTail;
case LoopVectorizeHints::FK_Disabled:
- return CM_EpilogueAllowed;
+ return LoopVectorizationUtils::EpilogueAllowed;
};
// 4) if the TTI hook indicates this is profitable, request tail-folding.
TailFoldingInfo TFI(TLI, &LVL, IAI);
if (TTI->preferTailFoldingOverEpilogue(&TFI))
- return CM_EpilogueNotNeededFoldTail;
+ return LoopVectorizationUtils::EpilogueNotNeededFoldTail;
- return CM_EpilogueAllowed;
+ return LoopVectorizationUtils::EpilogueAllowed;
}
// Process the loop in the VPlan-native vectorization path. This path builds
@@ -7576,7 +7465,7 @@ static bool processLoopInVPlanNativePath(
Function *F = L->getHeader()->getParent();
InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
- EpilogueLowering SEL =
+ LoopVectorizationUtils::EpilogueLowering SEL =
getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, *LVL, &IAI);
VFSelectionContext Config(*TTI, LVL, L, *F, PSE, ORE, &Hints, OptForSize);
@@ -7616,7 +7505,7 @@ static bool processLoopInVPlanNativePath(
hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
LVP.attachRuntimeChecks(BestPlan, Checks, HasBranchWeights);
- reportVectorization(ORE, L, VF, 1);
+ LoopVectorizationUtils::reportVectorization(LV_NAME, ORE, L, VF.Width, 1);
LVP.executePlan(VF.Width, /*UF=*/1, BestPlan, LB, DT);
}
@@ -7702,12 +7591,12 @@ static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
/// extra work when exiting the loop early, such as calculating the final
/// exit values of variables used outside the loop.
/// 3. The middle block.
-static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
- VectorizationFactor &VF, Loop *L,
- PredicatedScalarEvolution &PSE,
- VPCostContext &CostCtx, VPlan &Plan,
- EpilogueLowering SEL,
- std::optional<unsigned> VScale) {
+static bool
+isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF,
+ Loop *L, PredicatedScalarEvolution &PSE,
+ VPCostContext &CostCtx, VPlan &Plan,
+ LoopVectorizationUtils::EpilogueLowering SEL,
+ std::optional<unsigned> VScale) {
InstructionCost RtC = Checks.getCost();
if (!RtC.isValid())
return false;
@@ -7787,7 +7676,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// is allowed, choose the next closest multiple of VF. This should partly
// compensate for ignoring the epilogue cost.
uint64_t MinTC = std::max(MinTC1, MinTC2);
- if (SEL == CM_EpilogueAllowed)
+ if (SEL == LoopVectorizationUtils::EpilogueAllowed)
MinTC = alignTo(MinTC, IntVF);
VF.MinProfitableTripCount = ElementCount::getFixed(MinTC);
@@ -8296,9 +8185,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (LVL.hasUncountableEarlyExit()) {
if (!EnableEarlyExitVectorization) {
- reportVectorizationFailure("Auto-vectorization of loops with uncountable "
- "early exit is not enabled",
- "UncountableEarlyExitLoopsDisabled", ORE, L);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints.vectorizeAnalysisPassName(),
+ "Auto-vectorization of loops with uncountable "
+ "early exit is not enabled",
+ "UncountableEarlyExitLoopsDisabled", ORE, L);
return false;
}
}
@@ -8330,16 +8221,18 @@ bool LoopVectorizePass::processLoop(Loop *L) {
BasicBlock *LoopLatch = L->getLoopLatch();
if (IAI.requiresScalarEpilogue() ||
any_of(LVL.getCountableExitingBlocks(), not_equal_to(LoopLatch))) {
- reportVectorizationFailure("Auto-vectorization of early exit loops "
- "requiring a scalar epilogue is unsupported",
- "UncountableEarlyExitUnsupported", ORE, L);
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints.vectorizeAnalysisPassName(),
+ "Auto-vectorization of early exit loops "
+ "requiring a scalar epilogue is unsupported",
+ "UncountableEarlyExitUnsupported", ORE, L);
return false;
}
}
// Check the function attributes and profiles to find out if this function
// should be optimized for size.
- EpilogueLowering SEL =
+ LoopVectorizationUtils::EpilogueLowering SEL =
getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, LVL, &IAI);
// Check the loop for a trip count threshold: vectorize loops with a tiny trip
@@ -8356,19 +8249,20 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LLVM_DEBUG(dbgs() << "\n");
// Tail-folded loops are efficient even when the loop
// iteration count is low. However, setting the epilogue policy to
- // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
- // with runtime checks. It's more effective to let
+ // `LoopVectorizationUtils::EpilogueNotAllowedLowTripLoop` prevents
+ // vectorizing loops with runtime checks. It's more effective to let
// `isOutsideLoopWorkProfitable` determine if vectorization is
// beneficial for the loop.
- if (SEL != CM_EpilogueNotNeededFoldTail)
- SEL = CM_EpilogueNotAllowedLowTripLoop;
+ if (SEL != LoopVectorizationUtils::EpilogueNotNeededFoldTail)
+ SEL = LoopVectorizationUtils::EpilogueNotAllowedLowTripLoop;
}
}
// Check the function attributes to see if implicit floats or vectors are
// allowed.
if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints.vectorizeAnalysisPassName(),
"Can't vectorize when the NoImplicitFloat attribute is used",
"loop not vectorized due to NoImplicitFloat attribute",
"NoImplicitFloat", ORE, L);
@@ -8382,10 +8276,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// additional fp-math flags can help.
if (Hints.isPotentiallyUnsafe() &&
TTI->isFPVectorizationPotentiallyUnsafe()) {
- reportVectorizationFailure(
+ LoopVectorizationUtils::reportVectorizationFailure(
+ Hints.vectorizeAnalysisPassName(),
"Potentially unsafe FP op prevents vectorization",
- "loop not vectorized due to unsafe FP support.",
- "UnsafeFP", ORE, L);
+ "loop not vectorized due to unsafe FP support.", "UnsafeFP", ORE, L);
Hints.emitRemarkWithHints();
return false;
}
@@ -8592,7 +8486,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
});
} else {
// Report the vectorization decision.
- reportVectorization(ORE, L, VF, IC);
+ LoopVectorizationUtils::reportVectorization(LV_NAME, ORE, L, VF.Width, IC);
}
if (ORE->allowExtraAnalysis(LV_NAME))
checkMixedPrecision(L, ORE);
More information about the llvm-commits
mailing list