[llvm] [LV] Factor out VF-independent code from cost model (NFC). (PR #192426)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 20 06:15:51 PDT 2026
================
@@ -0,0 +1,531 @@
+//===- LoopVectorizationPlanner.cpp - VF selection and planning -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements VFSelectionContext methods for loop vectorization
+/// VF selection, independent of cost-modeling decisions.
+///
+//===----------------------------------------------------------------------===//
+
+#include "LoopVectorizationPlanner.h"
+#include "llvm/Analysis/CodeMetrics.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "loop-vectorize"
+
+static cl::opt<bool> MaximizeBandwidth(
+ "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
+ cl::desc("Maximize bandwidth when selecting vectorization factor which "
+ "will be determined by the smallest type in loop."));
+
+static cl::opt<bool> UseWiderVFIfCallVariantsPresent(
+ "vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true),
+ cl::Hidden,
+ cl::desc("Try wider VFs if they enable the use of vector variants"));
+
+static cl::opt<bool> ConsiderRegPressure(
+ "vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
+ cl::desc("Discard VFs if their register pressure is too high."));
+
+static cl::opt<bool> ForceTargetSupportsScalableVectors(
+ "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
+ cl::desc(
+ "Pretend that scalable vectors are supported, even if the target does "
+ "not support them. This flag should only be used for testing."));
+
+cl::opt<bool> llvm::PreferInLoopReductions(
+ "prefer-inloop-reductions", cl::init(false), cl::Hidden,
+ cl::desc("Prefer in-loop vector reductions, "
+ "overriding the targets preference."));
+
+bool VFSelectionContext::supportsScalableVectors() const {
+ return TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors;
+}
+
+bool VFSelectionContext::useMaxBandwidth(
+ TargetTransformInfo::RegisterKind RegKind) const {
+ return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 &&
+ (TTI.shouldMaximizeVectorBandwidth(RegKind) ||
+ (UseWiderVFIfCallVariantsPresent &&
+ Legal->hasVectorCallVariants())));
+}
+
+bool VFSelectionContext::shouldConsiderRegPressureForVF(ElementCount VF) const {
+ if (ConsiderRegPressure.getNumOccurrences())
+ return ConsiderRegPressure;
+
+ // TODO: We should eventually consider register pressure for all targets. The
+ // TTI hook is temporary whilst target-specific issues are being fixed.
+ if (TTI.shouldConsiderVectorizationRegPressure())
+ return true;
+
+ if (!useMaxBandwidth(VF.isScalable()
+ ? TargetTransformInfo::RGK_ScalableVector
+ : TargetTransformInfo::RGK_FixedWidthVector))
+ return false;
+ // Only calculate register pressure for VFs enabled by MaxBandwidth.
+ return ElementCount::isKnownGT(
+ VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
+ : MaxPermissibleVFWithoutMaxBW.FixedVF);
+}
+
+ElementCount VFSelectionContext::clampVFByMaxTripCount(
+ ElementCount VF, unsigned MaxTripCount, unsigned UserIC,
+ bool FoldTailByMasking, bool RequiresScalarEpilogue) const {
+ unsigned EstimatedVF = VF.getKnownMinValue();
+ if (VF.isScalable() && F.hasFnAttribute(Attribute::VScaleRange)) {
+ auto Attr = F.getFnAttribute(Attribute::VScaleRange);
+ auto Min = Attr.getVScaleRangeMin();
+ EstimatedVF *= Min;
+ }
+
+ // When a scalar epilogue is required, at least one iteration of the scalar
+ // loop has to execute. Adjust MaxTripCount accordingly to avoid picking a
+ // max VF that results in a dead vector loop.
+ if (MaxTripCount > 0 && RequiresScalarEpilogue)
+ MaxTripCount -= 1;
+
+ // When the user specifies an interleave count, we need to ensure that
+ // VF * UserIC <= MaxTripCount to avoid a dead vector loop.
+ unsigned IC = UserIC > 0 ? UserIC : 1;
+ unsigned EstimatedVFTimesIC = EstimatedVF * IC;
+
+ if (MaxTripCount && MaxTripCount <= EstimatedVFTimesIC &&
+ (!FoldTailByMasking || isPowerOf2_32(MaxTripCount))) {
+ // If upper bound loop trip count (TC) is known at compile time there is no
+ // point in choosing VF greater than TC / IC (as done in the loop below).
+ // Select maximum power of two which doesn't exceed TC / IC. If VF is
+ // scalable, we only fall back on a fixed VF when the TC is less than or
+ // equal to the known number of lanes.
+ auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount / IC);
+ if (ClampedUpperTripCount == 0)
+ ClampedUpperTripCount = 1;
+ LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
+ "exceeding the constant trip count"
+ << (UserIC > 0 ? " divided by UserIC" : "") << ": "
+ << ClampedUpperTripCount << "\n");
+ return ElementCount::get(ClampedUpperTripCount,
+ FoldTailByMasking ? VF.isScalable() : false);
+ }
+ return VF;
+}
+
+ElementCount VFSelectionContext::getMaximizedVFForTarget(
+ unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType,
+ ElementCount MaxSafeVF, unsigned UserIC, bool FoldTailByMasking,
+ bool RequiresScalarEpilogue) {
+ bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
+ const TypeSize WidestRegister = TTI.getRegisterBitWidth(
+ ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
+ : TargetTransformInfo::RGK_FixedWidthVector);
+
+ // Convenience function to return the minimum of two ElementCounts.
+ auto MinVF = [](const ElementCount &LHS, const ElementCount &RHS) {
+ assert((LHS.isScalable() == RHS.isScalable()) &&
+ "Scalable flags must match");
+ return ElementCount::isKnownLT(LHS, RHS) ? LHS : RHS;
+ };
+
+ // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
+ // Note that both WidestRegister and WidestType may not be a powers of 2.
+ auto MaxVectorElementCount = ElementCount::get(
+ llvm::bit_floor(WidestRegister.getKnownMinValue() / WidestType),
+ ComputeScalableMaxVF);
+ MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
+ LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "
+ << (MaxVectorElementCount * WidestType) << " bits.\n");
+
+ if (!MaxVectorElementCount) {
+ LLVM_DEBUG(dbgs() << "LV: The target has no "
+ << (ComputeScalableMaxVF ? "scalable" : "fixed")
+ << " vector registers.\n");
+ return ElementCount::getFixed(1);
+ }
+
+ ElementCount MaxVF =
+ clampVFByMaxTripCount(MaxVectorElementCount, MaxTripCount, UserIC,
+ FoldTailByMasking, RequiresScalarEpilogue);
+ // If the MaxVF was already clamped, there's no point in trying to pick a
+ // larger one.
+ if (MaxVF != MaxVectorElementCount)
+ return MaxVF;
+
+ TargetTransformInfo::RegisterKind RegKind =
+ ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
+ : TargetTransformInfo::RGK_FixedWidthVector;
+
+ if (MaxVF.isScalable())
+ MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
+ else
+ MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
+
+ if (useMaxBandwidth(RegKind)) {
+ auto MaxVectorElementCountMaxBW = ElementCount::get(
+ llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
+ ComputeScalableMaxVF);
+ MaxVF = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
+
+ if (ElementCount MinVF =
+ TTI.getMinimumVF(SmallestType, ComputeScalableMaxVF)) {
+ if (ElementCount::isKnownLT(MaxVF, MinVF)) {
+ LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVF
+ << ") with target's minimum: " << MinVF << '\n');
+ MaxVF = MinVF;
+ }
+ }
+
+ MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, UserIC,
+ FoldTailByMasking, RequiresScalarEpilogue);
+ }
+ return MaxVF;
+}
+
+namespace llvm {
+std::optional<unsigned> getMaxVScale(const Function &F,
+ const TargetTransformInfo &TTI) {
+ if (std::optional<unsigned> MaxVScale = TTI.getMaxVScale())
+ return MaxVScale;
+
+ if (F.hasFnAttribute(Attribute::VScaleRange))
+ return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
+
+ return std::nullopt;
+}
+
+#ifndef NDEBUG
+void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg,
+ Instruction *I) {
+ dbgs() << "LV: " << Prefix << DebugMsg;
+ if (I != nullptr)
+ dbgs() << " " << *I;
+ else
+ dbgs() << '.';
+ dbgs() << '\n';
+}
+#endif
+
+void reportVectorizationInfo(const char *PassName, const StringRef Msg,
----------------
david-arm wrote:
This looks different to the original version that was moved. Why the change?
https://github.com/llvm/llvm-project/pull/192426
More information about the llvm-commits
mailing list