[llvm] [LV] Factor out VF-independent code from cost model (NFC). (PR #192426)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 03:06:13 PDT 2026
================
@@ -3253,199 +3071,6 @@ bool LoopVectorizationCostModel::runtimeChecksRequired() {
return false;
}
-bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
- if (IsScalableVectorizationAllowed)
- return *IsScalableVectorizationAllowed;
-
- IsScalableVectorizationAllowed = false;
- if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors)
- return false;
-
- if (Hints->isScalableVectorizationDisabled()) {
- reportVectorizationInfo("Scalable vectorization is explicitly disabled",
- "ScalableVectorizationDisabled", ORE, TheLoop);
- return false;
- }
-
- LLVM_DEBUG(dbgs() << "LV: Scalable vectorization is available\n");
-
- auto MaxScalableVF = ElementCount::getScalable(
- std::numeric_limits<ElementCount::ScalarTy>::max());
-
- // Test that the loop-vectorizer can legalize all operations for this MaxVF.
- // FIXME: While for scalable vectors this is currently sufficient, this should
- // be replaced by a more detailed mechanism that filters out specific VFs,
- // instead of invalidating vectorization for a whole set of VFs based on the
- // MaxVF.
-
- // Disable scalable vectorization if the loop contains unsupported reductions.
- if (!canVectorizeReductions(MaxScalableVF)) {
- reportVectorizationInfo(
- "Scalable vectorization not supported for the reduction "
- "operations found in this loop.",
- "ScalableVFUnfeasible", ORE, TheLoop);
- return false;
- }
-
- // Disable scalable vectorization if the loop contains any instructions
- // with element types not supported for scalable vectors.
- if (any_of(ElementTypesInLoop, [&](Type *Ty) {
- return !Ty->isVoidTy() &&
- !this->TTI.isElementTypeLegalForScalableVector(Ty);
- })) {
- reportVectorizationInfo("Scalable vectorization is not supported "
- "for all element types found in this loop.",
- "ScalableVFUnfeasible", ORE, TheLoop);
- return false;
- }
-
- if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(*TheFunction, TTI)) {
- reportVectorizationInfo("The target does not provide maximum vscale value "
- "for safe distance analysis.",
- "ScalableVFUnfeasible", ORE, TheLoop);
- return false;
- }
-
- IsScalableVectorizationAllowed = true;
- return true;
-}
-
-ElementCount
-LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
- if (!isScalableVectorizationAllowed())
- return ElementCount::getScalable(0);
-
- auto MaxScalableVF = ElementCount::getScalable(
- std::numeric_limits<ElementCount::ScalarTy>::max());
- if (Legal->isSafeForAnyVectorWidth())
- return MaxScalableVF;
-
- std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
- // Limit MaxScalableVF by the maximum safe dependence distance.
- MaxScalableVF = ElementCount::getScalable(MaxSafeElements / *MaxVScale);
-
- if (!MaxScalableVF)
- reportVectorizationInfo(
- "Max legal vector width too small, scalable vectorization "
- "unfeasible.",
- "ScalableVFUnfeasible", ORE, TheLoop);
-
- return MaxScalableVF;
-}
-
-FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
- unsigned MaxTripCount, ElementCount UserVF, unsigned UserIC,
- bool FoldTailByMasking) {
- MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
- unsigned SmallestType, WidestType;
- std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
-
- // Get the maximum safe dependence distance in bits computed by LAA.
- // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
- // the memory accesses that is most restrictive (involved in the smallest
- // dependence distance).
- unsigned MaxSafeElementsPowerOf2 =
- bit_floor(Legal->getMaxSafeVectorWidthInBits() / WidestType);
- if (!Legal->isSafeForAnyStoreLoadForwardDistances()) {
- unsigned SLDist = Legal->getMaxStoreLoadForwardSafeDistanceInBits();
- MaxSafeElementsPowerOf2 =
- std::min(MaxSafeElementsPowerOf2, SLDist / WidestType);
- }
- auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
- auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
-
- if (!Legal->isSafeForAnyVectorWidth())
- this->MaxSafeElements = MaxSafeElementsPowerOf2;
-
- LLVM_DEBUG(dbgs() << "LV: The max safe fixed VF is: " << MaxSafeFixedVF
- << ".\n");
- LLVM_DEBUG(dbgs() << "LV: The max safe scalable VF is: " << MaxSafeScalableVF
- << ".\n");
-
- // First analyze the UserVF, fall back if the UserVF should be ignored.
- if (UserVF) {
- auto MaxSafeUserVF =
- UserVF.isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
-
- if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
- // If `VF=vscale x N` is safe, then so is `VF=N`
- if (UserVF.isScalable())
- return FixedScalableVFPair(
- ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
-
- return UserVF;
- }
-
- assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF));
-
- // Only clamp if the UserVF is not scalable. If the UserVF is scalable, it
- // is better to ignore the hint and let the compiler choose a suitable VF.
- if (!UserVF.isScalable()) {
- LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
- << " is unsafe, clamping to max safe VF="
- << MaxSafeFixedVF << ".\n");
- ORE->emit([&]() {
- return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
- TheLoop->getStartLoc(),
- TheLoop->getHeader())
- << "User-specified vectorization factor "
- << ore::NV("UserVectorizationFactor", UserVF)
- << " is unsafe, clamping to maximum safe vectorization factor "
- << ore::NV("VectorizationFactor", MaxSafeFixedVF);
- });
- return MaxSafeFixedVF;
- }
-
- if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors) {
- LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
- << " is ignored because scalable vectors are not "
- "available.\n");
- ORE->emit([&]() {
- return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
- TheLoop->getStartLoc(),
- TheLoop->getHeader())
- << "User-specified vectorization factor "
- << ore::NV("UserVectorizationFactor", UserVF)
- << " is ignored because the target does not support scalable "
- "vectors. The compiler will pick a more suitable value.";
- });
- } else {
- LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
- << " is unsafe. Ignoring scalable UserVF.\n");
- ORE->emit([&]() {
- return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
- TheLoop->getStartLoc(),
- TheLoop->getHeader())
- << "User-specified vectorization factor "
- << ore::NV("UserVectorizationFactor", UserVF)
- << " is unsafe. Ignoring the hint to let the compiler pick a "
- "more suitable value.";
- });
- }
- }
-
- LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType
- << " / " << WidestType << " bits.\n");
-
- FixedScalableVFPair Result(ElementCount::getFixed(1),
- ElementCount::getScalable(0));
- if (auto MaxVF =
- getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
- MaxSafeFixedVF, UserIC, FoldTailByMasking))
- Result.FixedVF = MaxVF;
-
- if (auto MaxVF =
- getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
- MaxSafeScalableVF, UserIC, FoldTailByMasking))
- if (MaxVF.isScalable()) {
- Result.ScalableVF = MaxVF;
- LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVF
- << "\n");
- }
-
- return Result;
-}
-
FixedScalableVFPair
LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
----------------
fhahn wrote:
We can move more things in the future, but the PR is already quite large. Things like tail-folding directly impact the cost/decisions per-VF, so it would probably better to keep it in LoopVectorizationCostModel for now
https://github.com/llvm/llvm-project/pull/192426
More information about the llvm-commits
mailing list