[llvm] ac2c986 - [LV][NFC] Rename ScalarEpilogueLowering to EpilogueLowering (#191871)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 05:55:00 PDT 2026
Author: Hassnaa Hamdi
Date: 2026-04-16T13:54:54+01:00
New Revision: ac2c986c5830b6c413ddbb0e75b8c1399fdffe85
URL: https://github.com/llvm/llvm-project/commit/ac2c986c5830b6c413ddbb0e75b8c1399fdffe85
DIFF: https://github.com/llvm/llvm-project/commit/ac2c986c5830b6c413ddbb0e75b8c1399fdffe85.diff
LOG: [LV][NFC] Rename ScalarEpilogueLowering to EpilogueLowering (#191871)
Rename ScalarEpilogueLowering enum to EpilogueLowering.
The term 'scalar' is misleading given that the epilogue could be
vectorized.
Also rename the enum values to use the tail-folding term instead of
predication, as it's more clear.
Added:
Modified:
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/include/llvm/CodeGen/BasicTTIImpl.h
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 3c4f00c0d87b5..0d2cf02831615 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -754,9 +754,9 @@ class TargetTransformInfo {
// vectorization should be considered.
LLVM_ABI unsigned getEpilogueVectorizationMinVF() const;
- /// Query the target whether it would be prefered to create a predicated
+ /// Query the target whether it would be preferred to create a tail-folded
/// vector loop, which can avoid the need to emit a scalar epilogue loop.
- LLVM_ABI bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const;
+ LLVM_ABI bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const;
/// Query the target what the preferred style of tail folding is.
LLVM_ABI TailFoldingStyle getPreferredTailFoldingStyle() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 7eb363c7b4404..3221f6b63e4f4 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -274,7 +274,7 @@ class TargetTransformInfoImplBase {
virtual unsigned getEpilogueVectorizationMinVF() const { return 16; }
- virtual bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
+ virtual bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
return false;
}
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 7dbd8bc658161..6dd3e264c4870 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -801,8 +801,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return BaseT::getEpilogueVectorizationMinVF();
}
- bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override {
- return BaseT::preferPredicateOverEpilogue(TFI);
+ bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override {
+ return BaseT::preferTailFoldingOverEpilogue(TFI);
}
TailFoldingStyle getPreferredTailFoldingStyle() const override {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 5111593d76a6d..28a418897a1d1 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -379,9 +379,9 @@ unsigned TargetTransformInfo::getEpilogueVectorizationMinVF() const {
return TTIImpl->getEpilogueVectorizationMinVF();
}
-bool TargetTransformInfo::preferPredicateOverEpilogue(
+bool TargetTransformInfo::preferTailFoldingOverEpilogue(
TailFoldingInfo *TFI) const {
- return TTIImpl->preferPredicateOverEpilogue(TFI);
+ return TTIImpl->preferTailFoldingOverEpilogue(TFI);
}
TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle() const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a9fe0c499ef6a..2947518f223f4 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6497,7 +6497,7 @@ unsigned AArch64TTIImpl::getEpilogueVectorizationMinVF() const {
return ST->getEpilogueVectorizationMinVF();
}
-bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
+bool AArch64TTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
if (!ST->hasSVE())
return false;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index cde391bdcaea8..db9d361b2d92a 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -473,7 +473,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
unsigned getEpilogueVectorizationMinVF() const override;
- bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override;
+ bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override;
bool supportsScalableVectors() const override {
return ST->isSVEorStreamingSVEAvailable();
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 03392fc2d84bc..c1df7fbb9d702 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -2616,14 +2616,14 @@ static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE,
return true;
}
-bool ARMTTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
+bool ARMTTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
if (!EnableTailPredication) {
- LLVM_DEBUG(dbgs() << "Tail-predication not enabled.\n");
+ LLVM_DEBUG(dbgs() << "Tail-folding not enabled.\n");
return false;
}
- // Creating a predicated vector loop is the first step for generating a
- // tail-predicated hardware loop, for which we need the MVE masked
+ // Creating a tail-folded vector loop is the first step for generating a
+ // tail-folded hardware loop, for which we need the MVE masked
// load/stores instructions:
if (!ST->hasMVEIntegerOps())
return false;
@@ -2633,17 +2633,18 @@ bool ARMTTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
// For now, restrict this to single block loops.
if (L->getNumBlocks() > 1) {
- LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: not a single block "
+ LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: not a single block "
"loop.\n");
return false;
}
- assert(L->isInnermost() && "preferPredicateOverEpilogue: inner-loop expected");
+ assert(L->isInnermost() &&
+ "preferTailFoldingOverEpilogue: inner-loop expected");
LoopInfo *LI = LVL->getLoopInfo();
HardwareLoopInfo HWLoopInfo(L);
if (!HWLoopInfo.canAnalyze(*LI)) {
- LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not "
+ LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
"analyzable.\n");
return false;
}
@@ -2654,14 +2655,14 @@ bool ARMTTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const {
// This checks if we have the low-overhead branch architecture
// extension, and if we will create a hardware-loop:
if (!isHardwareLoopProfitable(L, *SE, *AC, TFI->TLI, HWLoopInfo)) {
- LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not "
+ LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
"profitable.\n");
return false;
}
DominatorTree *DT = LVL->getDominatorTree();
if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT)) {
- LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not "
+ LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
"a candidate.\n");
return false;
}
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 0d6d5d202bddf..e824839e39159 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -436,7 +436,7 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
AssumptionCache &AC, TargetLibraryInfo *LibInfo,
HardwareLoopInfo &HWLoopInfo) const override;
- bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override;
+ bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override;
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index ad4d20bdc8956..a2a8385fc5359 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -121,7 +121,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool enableScalableVectorization() const override {
return ST->hasVInstructions();
}
- bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override {
+ bool preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const override {
return ST->hasVInstructions();
}
TailFoldingStyle getPreferredTailFoldingStyle() const override {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3bf3f599c9828..e20ef0296daea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -834,27 +834,27 @@ static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop,
namespace llvm {
-// Loop vectorization cost-model hints how the scalar epilogue loop should be
+// Loop vectorization cost-model hints how the epilogue/tail loop should be
// lowered.
-enum ScalarEpilogueLowering {
+enum EpilogueLowering {
- // The default: allowing scalar epilogues.
- CM_ScalarEpilogueAllowed,
+ // The default: allowing epilogues.
+ CM_EpilogueAllowed,
// Vectorization with OptForSize: don't allow epilogues.
- CM_ScalarEpilogueNotAllowedOptSize,
+ CM_EpilogueNotAllowedOptSize,
// A special case of vectorisation with OptForSize: loops with a very small
// trip count are considered for vectorization under OptForSize, thereby
// making sure the cost of their loop body is dominant, free of runtime
// guards and scalar iteration overheads.
- CM_ScalarEpilogueNotAllowedLowTripLoop,
+ CM_EpilogueNotAllowedLowTripLoop,
- // Loop hint predicate indicating an epilogue is undesired.
- CM_ScalarEpilogueNotNeededUsePredicate,
+ // Loop hint indicating an epilogue is undesired, apply tail folding.
+ CM_EpilogueNotNeededFoldTail,
- // Directive indicating we must either tail fold or not vectorize
- CM_ScalarEpilogueNotAllowedUsePredicate
+ // Directive indicating we must either fold the epilogue/tail or not vectorize
+ CM_EpilogueNotAllowedFoldTail
};
/// LoopVectorizationCostModel - estimates the expected speedups due to
@@ -868,7 +868,7 @@ class LoopVectorizationCostModel {
friend class LoopVectorizationPlanner;
public:
- LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L,
+ LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L,
PredicatedScalarEvolution &PSE, LoopInfo *LI,
LoopVectorizationLegality *Legal,
const TargetTransformInfo &TTI,
@@ -878,7 +878,7 @@ class LoopVectorizationCostModel {
std::function<BlockFrequencyInfo &()> GetBFI,
const Function *F, const LoopVectorizeHints *Hints,
InterleavedAccessInfo &IAI, bool OptForSize)
- : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
+ : EpilogueLoweringStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), GetBFI(GetBFI),
TheFunction(F), Hints(Hints), InterleaveInfo(IAI),
OptForSize(OptForSize) {
@@ -1285,7 +1285,7 @@ class LoopVectorizationCostModel {
/// Returns true if we're required to use a scalar epilogue for at least
/// the final iteration of the original loop.
bool requiresScalarEpilogue(bool IsVectorizing) const {
- if (!isScalarEpilogueAllowed()) {
+ if (!isEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
return false;
}
@@ -1306,16 +1306,16 @@ class LoopVectorizationCostModel {
return false;
}
- /// Returns true if a scalar epilogue is allowed (e.g.., not prevented by
+ /// Returns true if an epilogue is allowed (e.g., not prevented by
/// optsize or a loop hint annotation).
- bool isScalarEpilogueAllowed() const {
- return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
+ bool isEpilogueAllowed() const {
+ return EpilogueLoweringStatus == CM_EpilogueAllowed;
}
- /// Returns true if tail-folding is preferred over a scalar epilogue.
- bool preferPredicatedLoop() const {
- return ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate ||
- ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate;
+ /// Returns true if tail-folding is preferred over an epilogue.
+ bool preferTailFoldedLoop() const {
+ return EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail ||
+ EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail;
}
/// Returns the TailFoldingStyle that is best for the current loop.
@@ -1347,10 +1347,10 @@ class LoopVectorizationCostModel {
TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
if (EVLIsLegal)
return;
- // If for some reason EVL mode is unsupported, fallback to a scalar epilogue
+ // If for some reason EVL mode is unsupported, fallback to an epilogue
// if it's allowed, or DataWithoutLaneMask otherwise.
- if (ScalarEpilogueStatus == CM_ScalarEpilogueAllowed ||
- ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate)
+ if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
+ EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
ChosenTailFoldingStyle = TailFoldingStyle::None;
else
ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
@@ -1582,7 +1582,7 @@ class LoopVectorizationCostModel {
/// or as a peel-loop to handle gaps in interleave-groups.
/// Under optsize and when the trip count is very small we don't allow any
/// iterations to execute in the scalar loop.
- ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
+ EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
/// Control finally chosen tail folding style.
TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
@@ -2924,7 +2924,7 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
blockNeedsPredicationForAnyReason(I->getParent()) && isMaskRequired(I);
bool LoadAccessWithGapsRequiresEpilogMasking =
isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
- !isScalarEpilogueAllowed();
+ !isEpilogueAllowed();
bool StoreAccessWithGapsRequiresMasking =
isa<StoreInst>(I) && !Group->isFull();
if (!PredicatedAccessRequiresMasking &&
@@ -3461,7 +3461,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
ScalarEvolution *SE = PSE.getSE();
ElementCount TC = getSmallConstantTripCount(SE, TheLoop);
unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
- if (!MaxTC && ScalarEpilogueStatus == CM_ScalarEpilogueAllowed)
+ if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
MaxTC = getMaxTCFromNonZeroRange(PSE, TheLoop);
LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
if (TC != ElementCount::getFixed(MaxTC))
@@ -3489,25 +3489,23 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return FixedScalableVFPair::getNone();
}
- switch (ScalarEpilogueStatus) {
- case CM_ScalarEpilogueAllowed:
+ switch (EpilogueLoweringStatus) {
+ case CM_EpilogueAllowed:
return computeFeasibleMaxVF(MaxTC, UserVF, UserIC, false);
- case CM_ScalarEpilogueNotAllowedUsePredicate:
+ case CM_EpilogueNotAllowedFoldTail:
[[fallthrough]];
- case CM_ScalarEpilogueNotNeededUsePredicate:
- LLVM_DEBUG(
- dbgs() << "LV: vector predicate hint/switch found.\n"
- << "LV: Not allowing scalar epilogue, creating predicated "
- << "vector loop.\n");
+ case CM_EpilogueNotNeededFoldTail:
+ LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
+ << "LV: Not allowing epilogue, creating tail-folded "
+ << "vector loop.\n");
break;
- case CM_ScalarEpilogueNotAllowedLowTripLoop:
+ case CM_EpilogueNotAllowedLowTripLoop:
// fallthrough as a special case of OptForSize
- case CM_ScalarEpilogueNotAllowedOptSize:
- if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
- LLVM_DEBUG(
- dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
+ case CM_EpilogueNotAllowedOptSize:
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
+ LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
else
- LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "
+ LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
<< "count.\n");
// Bail if runtime checks are required, which are not good when optimising
@@ -3588,7 +3586,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// If we have a low-trip-count, and the fixed-width VF is known to divide
// the trip count but the scalable factor does not, use the fixed-width
// factor in preference to allow the generation of a non-predicated loop.
- if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedLowTripLoop &&
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
"remain for any chosen VF.\n");
@@ -3628,15 +3626,15 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
}
// If there was a tail-folding hint/switch, but we can't fold the tail by
- // masking, fallback to a vectorization with a scalar epilogue.
- if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
- LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
- "scalar epilogue instead.\n");
- ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
+ // masking, fallback to a vectorization with an epilogue.
+ if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
+ "epilogue instead.\n");
+ EpilogueLoweringStatus = CM_EpilogueAllowed;
return MaxFactors;
}
- if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
+ if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
return FixedScalableVFPair::getNone();
}
@@ -4213,7 +4211,7 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
return nullptr;
}
- if (!CM.isScalarEpilogueAllowed()) {
+ if (!CM.isEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
"epilogue is allowed.\n");
return nullptr;
@@ -4474,10 +4472,10 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Only interleave tail-folded loops if wide lane masks are requested, as the
// overhead of multiple instructions to calculate the predicate is likely
- // not beneficial. If a scalar epilogue is not allowed for any other reason,
+ // not beneficial. If an epilogue is not allowed for any other reason,
// do not interleave.
- if (!CM.isScalarEpilogueAllowed() &&
- !(CM.preferPredicatedLoop() && CM.useWideActiveLaneMask()))
+ if (!CM.isEpilogueAllowed() &&
+ !(CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask()))
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
@@ -4589,7 +4587,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
auto BestKnownTC =
getSmallBestKnownTC(PSE, OrigLoop,
/*CanUseConstantMax=*/true,
- /*CanExcludeZeroTrips=*/CM.isScalarEpilogueAllowed());
+ /*CanExcludeZeroTrips=*/CM.isEpilogueAllowed());
// For fixed length VFs treat a scalable trip count as unknown.
if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
@@ -4662,7 +4660,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
return IC;
}
- // For any scalar loop that either requires runtime checks or predication we
+ // For any scalar loop that either requires runtime checks or tail-folding we
// are better off leaving this to the unroller. Note that if we've already
// vectorized the loop we will have done the runtime check and so interleaving
// won't require further checks.
@@ -5239,7 +5237,7 @@ LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
// Calculate the cost of the whole interleaved group.
bool UseMaskForGaps =
- (Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed()) ||
+ (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
(isa<StoreInst>(I) && !Group->isFull());
InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
@@ -7943,7 +7941,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
- InterleaveGroups, RecipeBuilder, CM.isScalarEpilogueAllowed());
+ InterleaveGroups, RecipeBuilder, CM.isEpilogueAllowed());
// Replace VPValues for known constant strides.
RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *Plan, PSE,
@@ -8257,46 +8255,47 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(), PSE);
}
-// Determine how to lower the scalar epilogue, which depends on 1) optimising
-// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
-// predication, and 4) a TTI hook that analyses whether the loop is suitable
-// for predication.
-static ScalarEpilogueLowering getScalarEpilogueLowering(
- Function *F, Loop *L, LoopVectorizeHints &Hints, bool OptForSize,
- TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
- LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI) {
+// Determine how to lower the epilogue, which depends on 1) optimising
+// for minimum code-size, 2) tail-folding compiler options, 3) loop
+// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
+// is suitable for tail-folding.
+static EpilogueLowering
+getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
+ bool OptForSize, TargetTransformInfo *TTI,
+ TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL,
+ InterleavedAccessInfo *IAI) {
// 1) OptSize takes precedence over all other options, i.e. if this is set,
- // don't look at hints or options, and don't request a scalar epilogue.
+ // don't look at hints or options, and don't request an epilogue.
if (F->hasOptSize() ||
(OptForSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled))
- return CM_ScalarEpilogueNotAllowedOptSize;
+ return CM_EpilogueNotAllowedOptSize;
// 2) If set, obey the directives
if (PreferPredicateOverEpilogue.getNumOccurrences()) {
switch (PreferPredicateOverEpilogue) {
case PreferPredicateTy::ScalarEpilogue:
- return CM_ScalarEpilogueAllowed;
+ return CM_EpilogueAllowed;
case PreferPredicateTy::PredicateElseScalarEpilogue:
- return CM_ScalarEpilogueNotNeededUsePredicate;
+ return CM_EpilogueNotNeededFoldTail;
case PreferPredicateTy::PredicateOrDontVectorize:
- return CM_ScalarEpilogueNotAllowedUsePredicate;
+ return CM_EpilogueNotAllowedFoldTail;
};
}
// 3) If set, obey the hints
switch (Hints.getPredicate()) {
case LoopVectorizeHints::FK_Enabled:
- return CM_ScalarEpilogueNotNeededUsePredicate;
+ return CM_EpilogueNotNeededFoldTail;
case LoopVectorizeHints::FK_Disabled:
- return CM_ScalarEpilogueAllowed;
+ return CM_EpilogueAllowed;
};
- // 4) if the TTI hook indicates this is profitable, request predication.
+ // 4) if the TTI hook indicates this is profitable, request tail-folding.
TailFoldingInfo TFI(TLI, &LVL, IAI);
- if (TTI->preferPredicateOverEpilogue(&TFI))
- return CM_ScalarEpilogueNotNeededUsePredicate;
+ if (TTI->preferTailFoldingOverEpilogue(&TFI))
+ return CM_EpilogueNotNeededFoldTail;
- return CM_ScalarEpilogueAllowed;
+ return CM_EpilogueAllowed;
}
// Process the loop in the VPlan-native vectorization path. This path builds
@@ -8319,8 +8318,8 @@ static bool processLoopInVPlanNativePath(
Function *F = L->getHeader()->getParent();
InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
- ScalarEpilogueLowering SEL =
- getScalarEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, *LVL, &IAI);
+ EpilogueLowering SEL =
+ getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, *LVL, &IAI);
LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE,
GetBFI, F, &Hints, IAI, OptForSize);
@@ -8448,7 +8447,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
VectorizationFactor &VF, Loop *L,
PredicatedScalarEvolution &PSE,
VPCostContext &CostCtx, VPlan &Plan,
- ScalarEpilogueLowering SEL,
+ EpilogueLowering SEL,
std::optional<unsigned> VScale) {
InstructionCost RtC = Checks.getCost();
if (!RtC.isValid())
@@ -8525,11 +8524,11 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
uint64_t MinTC2 = divideCeil(RtC.getValue() * 10, ScalarC);
- // Now pick the larger minimum. If it is not a multiple of VF and a scalar
- // epilogue is allowed, choose the next closest multiple of VF. This should
- // partly compensate for ignoring the epilogue cost.
+ // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
+ // is allowed, choose the next closest multiple of VF. This should partly
+ // compensate for ignoring the epilogue cost.
uint64_t MinTC = std::max(MinTC1, MinTC2);
- if (SEL == CM_ScalarEpilogueAllowed)
+ if (SEL == CM_EpilogueAllowed)
MinTC = alignTo(MinTC, IntVF);
VF.MinProfitableTripCount = ElementCount::getFixed(MinTC);
@@ -9102,8 +9101,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check the function attributes and profiles to find out if this function
// should be optimized for size.
- ScalarEpilogueLowering SEL =
- getScalarEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, LVL, &IAI);
+ EpilogueLowering SEL =
+ getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, LVL, &IAI);
// Check the loop for a trip count threshold: vectorize loops with a tiny trip
// count by optimizing for size, to minimize overheads.
@@ -9117,14 +9116,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
else {
LLVM_DEBUG(dbgs() << "\n");
- // Predicate tail-folded loops are efficient even when the loop
+ // Tail-folded loops are efficient even when the loop
// iteration count is low. However, setting the epilogue policy to
- // `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
+ // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
// with runtime checks. It's more effective to let
// `isOutsideLoopWorkProfitable` determine if vectorization is
// beneficial for the loop.
- if (SEL != CM_ScalarEpilogueNotNeededUsePredicate)
- SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
+ if (SEL != CM_EpilogueNotNeededFoldTail)
+ SEL = CM_EpilogueNotAllowedLowTripLoop;
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d1ff014907dc9..d9a5d1841c7d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3683,7 +3683,7 @@ void VPlanTransforms::createInterleaveGroups(
VPlan &Plan,
const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
&InterleaveGroups,
- VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed) {
+ VPRecipeBuilder &RecipeBuilder, const bool &EpilogueAllowed) {
if (InterleaveGroups.empty())
return;
@@ -3710,7 +3710,7 @@ void VPlanTransforms::createInterleaveGroups(
}
bool NeedsMaskForGaps =
- (IG->requiresScalarEpilogue() && !ScalarEpilogueAllowed) ||
+ (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
(!StoredValues.empty() && !IG->isFull());
Instruction *IRInsertPos = IG->getInsertPos();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index bebf7ff1e9262..091e7f5c20dae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -334,7 +334,7 @@ struct VPlanTransforms {
VPlan &Plan,
const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
&InterleaveGroups,
- VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed);
+ VPRecipeBuilder &RecipeBuilder, const bool &EpilogueAllowed);
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
More information about the llvm-commits
mailing list