[llvm] [LV] Add -predicated-epilogue option for tail-folded epilogue (PR #190697)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 18:09:09 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/190697
>From 3efad40141ecfc39569119b899d2e31ec1df04fe Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 6 Apr 2026 19:04:29 +0000
Subject: [PATCH 1/2] [LV] Add -predicated-epilogue option for tail-folded
epilogue
This is the very beginning of implementing tail folding on
epilogue loops where we can have vectorized unpredicated main loop
along with vectorized predicated epilogue loop.
This patch is doing:
- Add a new value -predicated-epilogue to the opt-in option
-prefer-predicate-over-epilogue
- Add validation logic to filter out the cases where tail folding
will not be applied
- Add some logs
- Add tests for the excluded cases
This is just the foundational work: introducing the option and the
excluded cases. The actual implementation of tail folding on
epilogue loops will come gradually in future patches.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 110 +++++++++++---
.../LoopVectorize/no-epilog-tail-folding.ll | 140 ++++++++++++++++++
2 files changed, 231 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b268c81e550cf..1b2c161c8347d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -211,30 +211,35 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
// and predicate the instructions accordingly. If tail-folding fails, there are
// different fallback strategies depending on these values:
namespace PreferPredicateTy {
- enum Option {
- ScalarEpilogue = 0,
- PredicateElseScalarEpilogue,
- PredicateOrDontVectorize
- };
+// TODO: Rename enum values and flag names to use 'TailFold' instead of
+// 'Predicate' for clarity.
+enum Option {
+ ScalarEpilogue = 0,
+ PredicateElseScalarEpilogue,
+ PredicateOrDontVectorize,
+ PredicatedEpilogue
+};
} // namespace PreferPredicateTy
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"prefer-predicate-over-epilogue",
- cl::init(PreferPredicateTy::ScalarEpilogue),
- cl::Hidden,
+ cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,
cl::desc("Tail-folding and predication preferences over creating a scalar "
"epilogue loop."),
- cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
- "scalar-epilogue",
- "Don't tail-predicate loops, create scalar epilogue"),
- clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
- "predicate-else-scalar-epilogue",
- "prefer tail-folding, create scalar epilogue if tail "
- "folding fails."),
- clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
- "predicate-dont-vectorize",
- "prefers tail-folding, don't attempt vectorization if "
- "tail-folding fails.")));
+ cl::values(
+ clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue",
+ "Don't tail-predicate loops, create scalar epilogue"),
+ clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
+ "predicate-else-scalar-epilogue",
+ "prefer tail-folding, create scalar epilogue if tail "
+ "folding fails."),
+ clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
+ "predicate-dont-vectorize",
+ "prefers tail-folding, don't attempt vectorization if "
+ "tail-folding fails."),
+ clEnumValN(PreferPredicateTy::PredicatedEpilogue, "predicated-epilogue",
+ "prefers predicated vector epilogue, falling back on "
+ "scalar epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -899,7 +904,10 @@ enum ScalarEpilogueLowering {
// Loop hint predicate indicating an epilogue is undesired.
CM_ScalarEpilogueNotNeededUsePredicate,
-
+ // Predicated vector epilogue requested; the scalar epilogue of the
+ // vectorized epilogue loop will be tail-folded if possible, otherwise
+ // a scalar epilogue is kept.
+ CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
// Directive indicating we must either tail fold or not vectorize
CM_ScalarEpilogueNotAllowedUsePredicate
};
@@ -3647,6 +3655,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return FixedScalableVFPair::getNone();
break;
+ default:
+ // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ break;
}
// Now try the tail folding
@@ -8707,6 +8718,9 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
// predication, and 4) a TTI hook that analyses whether the loop is suitable
// for predication.
+// This function determines scalar epilogue lowering for the main vector loop
+// while scalar epilogue lowering for the predicated epilogue path is handled
+// separately in isEpilogueTailFoldingAllowed().
static ScalarEpilogueLowering getScalarEpilogueLowering(
Function *F, Loop *L, LoopVectorizeHints &Hints, bool OptForSize,
TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
@@ -8726,6 +8740,8 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
return CM_ScalarEpilogueNotNeededUsePredicate;
case PreferPredicateTy::PredicateOrDontVectorize:
return CM_ScalarEpilogueNotAllowedUsePredicate;
+ default:
+ break;
};
}
@@ -8745,6 +8761,58 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
return CM_ScalarEpilogueAllowed;
}
+/// Check if we can apply tail-folding on the scalar loop of the vectorized
+/// epilogue loop, so that we can have vectorized unpredicated main loop along
+/// with vectorized predicated epilogue loop.
+static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
+ const LoopVectorizationLegality &LVL) {
+ // Epilogue TF is only enabled when explicitly requested via command line.
+ if (!PreferPredicateOverEpilogue.getNumOccurrences() ||
+ PreferPredicateOverEpilogue != PreferPredicateTy::PredicatedEpilogue)
+ return false;
+
+ if (!EnableEpilogueVectorization) {
+ LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
+ "disallowed while epilogue predication allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ return false;
+ }
+
+ // If scalar epilogue is explicitly required, we can't apply TF.
+ if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ "scalar epilogue is required\n");
+ return false;
+ }
+
+ // If having epilogue is NOT allowed, then no epilogue to apply TF for.
+ if (!CM.isScalarEpilogueAllowed()) {
+ LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail folding for.\n");
+ return false;
+ }
+
+ // TF is not supported for some kinds of reductions that results in more
+ // overhead
+ bool HasReductions = !LVL.getReductionVars().empty();
+ bool HasSelectCmpReductions =
+ HasReductions &&
+ any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
+ const RecurrenceDescriptor &RdxDesc = Reduction.second;
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
+ });
+ if (HasSelectCmpReductions) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
+ "select-cmp Reductions\n");
+ return false;
+ }
+
+ // We can apply TF on the scalar loop of the vectorized epilogue.
+ return true;
+}
+
// Process the loop in the VPlan-native vectorization path. This path builds
// VPlan upfront in the vectorization pipeline, which allows to apply
// VPlan-to-VPlan transformations from the very beginning without modifying the
@@ -9615,6 +9683,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
ORE);
+ if (isEpilogueTailFoldingAllowed(CM, LVL)) {
+ // TODO: Apply tail folding on the vectorized epilogue loop.
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ }
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
new file mode 100644
index 0000000000000..7db85bb2f739e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -0,0 +1,140 @@
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
+; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
+; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+define void @test_epilogue_tf(ptr %a, i64 %n) {
+; CHECK: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK: LV: Epilogue tail folding is enabled
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+define void @epilogue_is_disabled(ptr %a, i64 %n) {
+; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+ at AB = common global [1024 x i32] zeroinitializer, align 4
+ at CD = common global [1024 x i32] zeroinitializer, align 4
+define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
+; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
+; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv
+ %tmp = load i32, ptr %arrayidx0, align 4
+ %tmp1 = or disjoint i64 %indvars.iv, 1
+ %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp1
+ %tmp2 = load i32, ptr %arrayidx1, align 4
+ %add = add nsw i32 %tmp, %C
+ %mul = mul nsw i32 %tmp2, %D
+ %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx2, align 4
+ %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp1
+ store i32 %mul, ptr %arrayidx3, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+ %cmp = icmp slt i64 %indvars.iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body
+ ret void
+}
+
+define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
+; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
+; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %loop.header
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i64 0, ptr %gep
+ br i1 true, label %loop.then, label %exit.2
+loop.then:
+ %cmp3 = icmp ne i64 %iv, %x
+ br i1 %cmp3, label %loop.latch, label %exit.1
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ br label %loop.header
+exit.1:
+ ret i16 0
+exit.2:
+ ret i16 1
+}
+
+define i32 @opt_for_size(ptr %p, i32 %n) optsize {
+; CHECK: LV: Checking a loop in 'opt_for_size'
+; CHECK: LV: No epilogue to apply tail folding for.
+entry:
+ br label %for.body
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
+ %0 = load i32, ptr %arrayidx, align 1
+ %cmp1 = icmp eq i32 %0, 0
+ %sel = select i1 %cmp1, i32 2, i32 1
+ store i32 %sel, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.end, label %for.body
+for.end:
+ ret i32 0
+}
+
+define i32 @max_reduction_epilog(ptr %src, i64 %N) {
+; CHECK: LV: Checking a loop in 'max_reduction_epilog'
+; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %icmp = icmp ugt i32 %load, %red
+ %select = select i1 %icmp, i32 %load, i32 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %N
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %select
+}
>From 2295f7572ea9cbee6be22fdf5e73148982e7d0e4 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 01:08:38 +0000
Subject: [PATCH 2/2] Add 'REQUIRES: asserts' to the test file.
---
llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index 7db85bb2f739e..e717b59935520 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,3 +1,4 @@
+; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
More information about the llvm-commits
mailing list