[llvm] [LV] Consider UserIC when limiting VF. (PR #174573)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 20 05:49:31 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/174573
>From a6631cfa9410d917c5eae4ba56af2eb8d0aadf00 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 6 Jan 2026 10:35:59 +0000
Subject: [PATCH] [LV] Consider UserIC when limiting VF.
If a UserIC is provided, the vector loop will process VF * UserIC. Pass
it through UserIC to computeFeasibleMaxVF and use it to limit the max VF
to factors where VF * UserIC <= MaxTripCount. This avoids creating dead
vector loops with user provided interleave counts.
---
.../Vectorize/LoopVectorizationPlanner.h | 2 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 51 ++++++++------
.../AArch64/epilog-vectorization-factors.ll | 66 ++++++++++---------
3 files changed, 68 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 430d9469c7698..2ff3ced714d46 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -566,7 +566,7 @@ class LoopVectorizationPlanner {
/// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if
/// epilogue vectorization is not supported for the loop.
VectorizationFactor
- selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC);
+ selectEpilogueVectorizationFactor(const ElementCount MainLoopVF, unsigned IC);
/// Emit remarks for recipes with invalid costs in the available VPlans.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 31c44fc46c973..52502e5f63a3f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1503,12 +1503,13 @@ class LoopVectorizationCostModel {
/// disabled or unsupported, then the scalable part will be equal to
/// ElementCount::getScalable(0).
FixedScalableVFPair computeFeasibleMaxVF(unsigned MaxTripCount,
- ElementCount UserVF,
+ ElementCount UserVF, unsigned UserIC,
bool FoldTailByMasking);
- /// If \p VF > MaxTripcount, clamps it to the next lower VF that is <=
- /// MaxTripCount.
+ /// If \p VF * \p UserIC > MaxTripcount, clamps VF to the next lower VF that
+ /// results in VF * UserIC <= MaxTripCount.
ElementCount clampVFByMaxTripCount(ElementCount VF, unsigned MaxTripCount,
+ unsigned UserIC,
bool FoldTailByMasking) const;
/// \return the maximized element count based on the targets vector
@@ -1517,7 +1518,7 @@ class LoopVectorizationCostModel {
ElementCount getMaximizedVFForTarget(unsigned MaxTripCount,
unsigned SmallestType,
unsigned WidestType,
- ElementCount MaxSafeVF,
+ ElementCount MaxSafeVF, unsigned UserIC,
bool FoldTailByMasking);
/// Checks if scalable vectorization is supported and enabled. Caches the
@@ -3445,7 +3446,8 @@ LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
}
FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
- unsigned MaxTripCount, ElementCount UserVF, bool FoldTailByMasking) {
+ unsigned MaxTripCount, ElementCount UserVF, unsigned UserIC,
+ bool FoldTailByMasking) {
MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
unsigned SmallestType, WidestType;
std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
@@ -3541,12 +3543,12 @@ FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
ElementCount::getScalable(0));
if (auto MaxVF =
getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
- MaxSafeFixedVF, FoldTailByMasking))
+ MaxSafeFixedVF, UserIC, FoldTailByMasking))
Result.FixedVF = MaxVF;
if (auto MaxVF =
getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
- MaxSafeScalableVF, FoldTailByMasking))
+ MaxSafeScalableVF, UserIC, FoldTailByMasking))
if (MaxVF.isScalable()) {
Result.ScalableVF = MaxVF;
LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVF
@@ -3599,7 +3601,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
switch (ScalarEpilogueStatus) {
case CM_ScalarEpilogueAllowed:
- return computeFeasibleMaxVF(MaxTC, UserVF, false);
+ return computeFeasibleMaxVF(MaxTC, UserVF, UserIC, false);
case CM_ScalarEpilogueNotAllowedUsePredicate:
[[fallthrough]];
case CM_ScalarEpilogueNotNeededUsePredicate:
@@ -3638,7 +3640,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
}
- FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(MaxTC, UserVF, true);
+ FixedScalableVFPair MaxFactors =
+ computeFeasibleMaxVF(MaxTC, UserVF, UserIC, true);
// Avoid tail folding if the trip count is known to be a multiple of any VF
// we choose.
@@ -3793,7 +3796,8 @@ bool LoopVectorizationCostModel::useMaxBandwidth(
}
ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
- ElementCount VF, unsigned MaxTripCount, bool FoldTailByMasking) const {
+ ElementCount VF, unsigned MaxTripCount, unsigned UserIC,
+ bool FoldTailByMasking) const {
unsigned EstimatedVF = VF.getKnownMinValue();
if (VF.isScalable() && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
auto Attr = TheFunction->getFnAttribute(Attribute::VScaleRange);
@@ -3807,16 +3811,24 @@ ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
if (MaxTripCount > 0 && requiresScalarEpilogue(true))
MaxTripCount -= 1;
- if (MaxTripCount && MaxTripCount <= EstimatedVF &&
+ // When the user specifies an interleave count, we need to ensure that
+ // VF * UserIC <= MaxTripCount to avoid a dead vector loop.
+ unsigned IC = UserIC > 0 ? UserIC : 1;
+ unsigned EstimatedVFTimesIC = EstimatedVF * IC;
+
+ if (MaxTripCount && MaxTripCount <= EstimatedVFTimesIC &&
(!FoldTailByMasking || isPowerOf2_32(MaxTripCount))) {
// If upper bound loop trip count (TC) is known at compile time there is no
- // point in choosing VF greater than TC (as done in the loop below). Select
- // maximum power of two which doesn't exceed TC. If VF is
+ // point in choosing VF greater than TC / IC (as done in the loop below).
+ // Select maximum power of two which doesn't exceed TC / IC. If VF is
// scalable, we only fall back on a fixed VF when the TC is less than or
// equal to the known number of lanes.
- auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount);
+ auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount / IC);
+ if (ClampedUpperTripCount == 0)
+ ClampedUpperTripCount = 1;
LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
- "exceeding the constant trip count: "
+ "exceeding the constant trip count"
+ << (UserIC > 0 ? " divided by UserIC" : "") << ": "
<< ClampedUpperTripCount << "\n");
return ElementCount::get(ClampedUpperTripCount,
FoldTailByMasking ? VF.isScalable() : false);
@@ -3826,7 +3838,7 @@ ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType,
- ElementCount MaxSafeVF, bool FoldTailByMasking) {
+ ElementCount MaxSafeVF, unsigned UserIC, bool FoldTailByMasking) {
bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
const TypeSize WidestRegister = TTI.getRegisterBitWidth(
ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
@@ -3855,8 +3867,8 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
return ElementCount::getFixed(1);
}
- ElementCount MaxVF = clampVFByMaxTripCount(MaxVectorElementCount,
- MaxTripCount, FoldTailByMasking);
+ ElementCount MaxVF = clampVFByMaxTripCount(
+ MaxVectorElementCount, MaxTripCount, UserIC, FoldTailByMasking);
// If the MaxVF was already clamped, there's no point in trying to pick a
// larger one.
if (MaxVF != MaxVectorElementCount)
@@ -3886,7 +3898,8 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
}
}
- MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, FoldTailByMasking);
+ MaxVF =
+ clampVFByMaxTripCount(MaxVF, MaxTripCount, UserIC, FoldTailByMasking);
if (MaxVectorElementCount != MaxVF) {
// Invalidate any widening decisions we might have made, in case the loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
index 549df337e6907..6cf8dcabfa913 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
@@ -331,7 +331,6 @@ exit:
ret void
}
-; TODO: Choose smaller VF * UF for main loop, so we do not create a dead vector loop.
define void @small_trip_count_loop(ptr %arg, ptr %arg2) {
; CHECK-LABEL: @small_trip_count_loop(
; CHECK-NEXT: iter.check:
@@ -340,53 +339,58 @@ define void @small_trip_count_loop(ptr %arg, ptr %arg2) {
; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[ARG21]], [[ARG3]]
-; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 64
+; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
; CHECK: vector.main.loop.iter.check:
-; CHECK-NEXT: br i1 true, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 16
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 32
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 48
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[ARG]], align 1
-; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = add <16 x i8> [[WIDE_LOAD]], splat (i8 10)
-; CHECK-NEXT: [[TMP5:%.*]] = add <16 x i8> [[WIDE_LOAD4]], splat (i8 10)
-; CHECK-NEXT: [[TMP6:%.*]] = add <16 x i8> [[WIDE_LOAD5]], splat (i8 10)
-; CHECK-NEXT: [[TMP7:%.*]] = add <16 x i8> [[WIDE_LOAD6]], splat (i8 10)
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 16
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 32
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 48
-; CHECK-NEXT: store <16 x i8> [[TMP4]], ptr [[ARG2]], align 1
-; CHECK-NEXT: store <16 x i8> [[TMP5]], ptr [[TMP8]], align 1
-; CHECK-NEXT: store <16 x i8> [[TMP6]], ptr [[TMP9]], align 1
-; CHECK-NEXT: store <16 x i8> [[TMP7]], ptr [[TMP10]], align 1
-; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[INDEX1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 4
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 12
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP14]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 10)
+; CHECK-NEXT: [[TMP15:%.*]] = add <4 x i8> [[WIDE_LOAD4]], splat (i8 10)
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i8> [[WIDE_LOAD7]], splat (i8 10)
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i8> [[WIDE_LOAD6]], splat (i8 10)
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[INDEX1]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 4
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 8
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 12
+; CHECK-NEXT: store <4 x i8> [[TMP5]], ptr [[TMP10]], align 1
+; CHECK-NEXT: store <4 x i8> [[TMP15]], ptr [[TMP17]], align 1
+; CHECK-NEXT: store <4 x i8> [[TMP7]], ptr [[TMP18]], align 1
+; CHECK-NEXT: store <4 x i8> [[TMP16]], ptr [[TMP12]], align 1
+; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i32 [[INDEX1]], 16
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT1]], 16
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF14:![0-9]+]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11]]
; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 0, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
-; CHECK-NEXT: [[TMP12:%.*]] = add <16 x i8> [[WIDE_LOAD7]], splat (i8 10)
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i8> [[WIDE_LOAD5]], splat (i8 10)
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[INDEX]]
-; CHECK-NEXT: store <16 x i8> [[TMP12]], ptr [[TMP13]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
-; CHECK-NEXT: br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: store <4 x i8> [[TMP6]], ptr [[TMP13]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20
+; CHECK-NEXT: br i1 [[TMP8]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 16, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
More information about the llvm-commits
mailing list