[llvm] [LV] Add -predicated-epilogue option for tail-folded epilogue (PR #190697)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 05:35:45 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/190697
>From 6fc3bbdf814fcff8723ded4747d9de8e50496561 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 6 Apr 2026 19:04:29 +0000
Subject: [PATCH 1/5] [LV] Add -predicated-epilogue option for tail-folded
epilogue
This is the very beginning of implementing tail folding on
epilogue loops where we can have vectorized unpredicated main loop
along with vectorized predicated epilogue loop.
This patch is doing:
- Add a new value -predicated-epilogue to the opt-in option
-prefer-predicate-over-epilogue
- Add validation logic to filter out the cases where tail folding
will not be applied
- Add some logs
- Add tests for the excluded cases
This is just the foundational work: introducing the option and the
excluded cases. The actual implementation of tail folding on
epilogue loops will come gradually in future patches.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 110 +++++++++++---
.../LoopVectorize/no-epilog-tail-folding.ll | 140 ++++++++++++++++++
2 files changed, 231 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3bf3f599c9828..4697a67214ce8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -211,30 +211,35 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
// and predicate the instructions accordingly. If tail-folding fails, there are
// different fallback strategies depending on these values:
namespace PreferPredicateTy {
- enum Option {
- ScalarEpilogue = 0,
- PredicateElseScalarEpilogue,
- PredicateOrDontVectorize
- };
+// TODO: Rename enum values and flag names to use 'TailFold' instead of
+// 'Predicate' for clarity.
+enum Option {
+ ScalarEpilogue = 0,
+ PredicateElseScalarEpilogue,
+ PredicateOrDontVectorize,
+ PredicatedEpilogue
+};
} // namespace PreferPredicateTy
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"prefer-predicate-over-epilogue",
- cl::init(PreferPredicateTy::ScalarEpilogue),
- cl::Hidden,
+ cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,
cl::desc("Tail-folding and predication preferences over creating a scalar "
"epilogue loop."),
- cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
- "scalar-epilogue",
- "Don't tail-predicate loops, create scalar epilogue"),
- clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
- "predicate-else-scalar-epilogue",
- "prefer tail-folding, create scalar epilogue if tail "
- "folding fails."),
- clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
- "predicate-dont-vectorize",
- "prefers tail-folding, don't attempt vectorization if "
- "tail-folding fails.")));
+ cl::values(
+ clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue",
+ "Don't tail-predicate loops, create scalar epilogue"),
+ clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
+ "predicate-else-scalar-epilogue",
+ "prefer tail-folding, create scalar epilogue if tail "
+ "folding fails."),
+ clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
+ "predicate-dont-vectorize",
+ "prefers tail-folding, don't attempt vectorization if "
+ "tail-folding fails."),
+ clEnumValN(PreferPredicateTy::PredicatedEpilogue, "predicated-epilogue",
+ "prefers predicated vector epilogue, falling back on "
+ "scalar epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -852,7 +857,10 @@ enum ScalarEpilogueLowering {
// Loop hint predicate indicating an epilogue is undesired.
CM_ScalarEpilogueNotNeededUsePredicate,
-
+ // Predicated vector epilogue requested; the scalar epilogue of the
+ // vectorized epilogue loop will be tail-folded if possible, otherwise
+ // a scalar epilogue is kept.
+ CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
// Directive indicating we must either tail fold or not vectorize
CM_ScalarEpilogueNotAllowedUsePredicate
};
@@ -3516,6 +3524,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return FixedScalableVFPair::getNone();
break;
+ default:
+ // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ break;
}
// Now try the tail folding
@@ -8261,6 +8272,9 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
// predication, and 4) a TTI hook that analyses whether the loop is suitable
// for predication.
+// This function determines scalar epilogue lowering for the main vector loop
+// while scalar epilogue lowering for the predicated epilogue path is handled
+// separately in isEpilogueTailFoldingAllowed().
static ScalarEpilogueLowering getScalarEpilogueLowering(
Function *F, Loop *L, LoopVectorizeHints &Hints, bool OptForSize,
TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
@@ -8280,6 +8294,8 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
return CM_ScalarEpilogueNotNeededUsePredicate;
case PreferPredicateTy::PredicateOrDontVectorize:
return CM_ScalarEpilogueNotAllowedUsePredicate;
+ default:
+ break;
};
}
@@ -8299,6 +8315,58 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
return CM_ScalarEpilogueAllowed;
}
+/// Check if we can apply tail-folding on the scalar loop of the vectorized
+/// epilogue loop, so that we can have vectorized unpredicated main loop along
+/// with vectorized predicated epilogue loop.
+static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
+ const LoopVectorizationLegality &LVL) {
+ // Epilogue TF is only enabled when explicitly requested via command line.
+ if (!PreferPredicateOverEpilogue.getNumOccurrences() ||
+ PreferPredicateOverEpilogue != PreferPredicateTy::PredicatedEpilogue)
+ return false;
+
+ if (!EnableEpilogueVectorization) {
+ LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
+ "disallowed while epilogue predication allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ return false;
+ }
+
+ // If scalar epilogue is explicitly required, we can't apply TF.
+ if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ "scalar epilogue is required\n");
+ return false;
+ }
+
+ // If having epilogue is NOT allowed, then no epilogue to apply TF for.
+ if (!CM.isScalarEpilogueAllowed()) {
+ LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail folding for.\n");
+ return false;
+ }
+
+ // TF is not supported for some kinds of reductions that results in more
+ // overhead
+ bool HasReductions = !LVL.getReductionVars().empty();
+ bool HasSelectCmpReductions =
+ HasReductions &&
+ any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
+ const RecurrenceDescriptor &RdxDesc = Reduction.second;
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
+ });
+ if (HasSelectCmpReductions) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
+ "select-cmp Reductions\n");
+ return false;
+ }
+
+ // We can apply TF on the scalar loop of the vectorized epilogue.
+ return true;
+}
+
// Process the loop in the VPlan-native vectorization path. This path builds
// VPlan upfront in the vectorization pipeline, which allows to apply
// VPlan-to-VPlan transformations from the very beginning without modifying the
@@ -9181,6 +9249,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
ORE);
+ if (isEpilogueTailFoldingAllowed(CM, LVL)) {
+ // TODO: Apply tail folding on the vectorized epilogue loop.
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ }
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
new file mode 100644
index 0000000000000..7db85bb2f739e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -0,0 +1,140 @@
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
+; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
+; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+define void @test_epilogue_tf(ptr %a, i64 %n) {
+; CHECK: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK: LV: Epilogue tail folding is enabled
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+define void @epilogue_is_disabled(ptr %a, i64 %n) {
+; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+ at AB = common global [1024 x i32] zeroinitializer, align 4
+ at CD = common global [1024 x i32] zeroinitializer, align 4
+define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
+; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
+; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv
+ %tmp = load i32, ptr %arrayidx0, align 4
+ %tmp1 = or disjoint i64 %indvars.iv, 1
+ %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp1
+ %tmp2 = load i32, ptr %arrayidx1, align 4
+ %add = add nsw i32 %tmp, %C
+ %mul = mul nsw i32 %tmp2, %D
+ %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx2, align 4
+ %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp1
+ store i32 %mul, ptr %arrayidx3, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+ %cmp = icmp slt i64 %indvars.iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body
+ ret void
+}
+
+define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
+; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
+; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %loop.header
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i64 0, ptr %gep
+ br i1 true, label %loop.then, label %exit.2
+loop.then:
+ %cmp3 = icmp ne i64 %iv, %x
+ br i1 %cmp3, label %loop.latch, label %exit.1
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ br label %loop.header
+exit.1:
+ ret i16 0
+exit.2:
+ ret i16 1
+}
+
+define i32 @opt_for_size(ptr %p, i32 %n) optsize {
+; CHECK: LV: Checking a loop in 'opt_for_size'
+; CHECK: LV: No epilogue to apply tail folding for.
+entry:
+ br label %for.body
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
+ %0 = load i32, ptr %arrayidx, align 1
+ %cmp1 = icmp eq i32 %0, 0
+ %sel = select i1 %cmp1, i32 2, i32 1
+ store i32 %sel, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.end, label %for.body
+for.end:
+ ret i32 0
+}
+
+define i32 @max_reduction_epilog(ptr %src, i64 %N) {
+; CHECK: LV: Checking a loop in 'max_reduction_epilog'
+; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %icmp = icmp ugt i32 %load, %red
+ %select = select i1 %icmp, i32 %load, i32 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %N
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %select
+}
>From b9c15d78fb4cf344912aeafa2788951f6ccbe03b Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 01:08:38 +0000
Subject: [PATCH 2/5] Add 'REQUIRES: asserts' to the test file.
---
llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index 7db85bb2f739e..e717b59935520 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,3 +1,4 @@
+; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
>From 2ba0624d9895a131b2053972148d32decfc703cb Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 14:47:25 +0000
Subject: [PATCH 3/5] improve readability and rebase
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4697a67214ce8..d67ca3381eb2d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -857,10 +857,12 @@ enum ScalarEpilogueLowering {
// Loop hint predicate indicating an epilogue is undesired.
CM_ScalarEpilogueNotNeededUsePredicate,
+
// Predicated vector epilogue requested; the scalar epilogue of the
// vectorized epilogue loop will be tail-folded if possible, otherwise
// a scalar epilogue is kept.
CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
+
// Directive indicating we must either tail fold or not vectorize
CM_ScalarEpilogueNotAllowedUsePredicate
};
@@ -8345,8 +8347,8 @@ static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
return false;
}
- // TF is not supported for some kinds of reductions that results in more
- // overhead
+ // TF is not supported for some kinds of reductions that result in more
+ // overhead.
bool HasReductions = !LVL.getReductionVars().empty();
bool HasSelectCmpReductions =
HasReductions &&
>From ba8cadfdf43ef49baba9ac0603557988423d86e6 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 13 Apr 2026 21:04:46 +0000
Subject: [PATCH 4/5] resolve review comments - improve readability
---
.../Transforms/Vectorize/LoopVectorize.cpp | 70 ++++++++-----------
.../LoopVectorize/no-epilog-tail-folding.ll | 19 ++---
2 files changed, 40 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d67ca3381eb2d..53f00de0e30e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -211,14 +211,12 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
// and predicate the instructions accordingly. If tail-folding fails, there are
// different fallback strategies depending on these values:
namespace PreferPredicateTy {
-// TODO: Rename enum values and flag names to use 'TailFold' instead of
-// 'Predicate' for clarity.
-enum Option {
- ScalarEpilogue = 0,
- PredicateElseScalarEpilogue,
- PredicateOrDontVectorize,
- PredicatedEpilogue
-};
+ enum Option {
+ ScalarEpilogue = 0,
+ PredicateElseScalarEpilogue,
+ PredicateOrDontVectorize,
+ FoldEpilogueTail
+ };
} // namespace PreferPredicateTy
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
@@ -237,9 +235,9 @@ static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"predicate-dont-vectorize",
"prefers tail-folding, don't attempt vectorization if "
"tail-folding fails."),
- clEnumValN(PreferPredicateTy::PredicatedEpilogue, "predicated-epilogue",
- "prefers predicated vector epilogue, falling back on "
- "scalar epilogue if it fails.")));
+ clEnumValN(PreferPredicateTy::FoldEpilogueTail, "fold-epilogue-tail",
+ "prefers tail-folded vector epilogue, falling back on "
+ "an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -858,10 +856,10 @@ enum ScalarEpilogueLowering {
// Loop hint predicate indicating an epilogue is undesired.
CM_ScalarEpilogueNotNeededUsePredicate,
- // Predicated vector epilogue requested; the scalar epilogue of the
- // vectorized epilogue loop will be tail-folded if possible, otherwise
- // a scalar epilogue is kept.
- CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
+ // Tail-folded vector epilogue requested; the scalar tail will be folded into
+ // the vectorized epilogue loop if possible, otherwise fall back on a scalar
+ // epilogue.
+ CM_EpilogueNotNeededFoldEpilogueTail,
// Directive indicating we must either tail fold or not vectorize
CM_ScalarEpilogueNotAllowedUsePredicate
@@ -3527,7 +3525,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
break;
default:
- // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ // TODO: handle the case for CM_EpilogueNotNeededFoldEpilogueTail
break;
}
@@ -8275,7 +8273,7 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// predication, and 4) a TTI hook that analyses whether the loop is suitable
// for predication.
// This function determines scalar epilogue lowering for the main vector loop
-// while scalar epilogue lowering for the predicated epilogue path is handled
+// while scalar epilogue lowering for the tail-folded epilogue path is handled
// separately in isEpilogueTailFoldingAllowed().
static ScalarEpilogueLowering getScalarEpilogueLowering(
Function *F, Loop *L, LoopVectorizeHints &Hints, bool OptForSize,
@@ -8317,26 +8315,26 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
return CM_ScalarEpilogueAllowed;
}
-/// Check if we can apply tail-folding on the scalar loop of the vectorized
-/// epilogue loop, so that we can have vectorized unpredicated main loop along
-/// with vectorized predicated epilogue loop.
+/// Check if we can apply tail folding to the vectorized epilogue loop,
+/// enabling an unpredicated main vector loop with a tail-folded epilogue
+/// vector loop.
static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
const LoopVectorizationLegality &LVL) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!PreferPredicateOverEpilogue.getNumOccurrences() ||
- PreferPredicateOverEpilogue != PreferPredicateTy::PredicatedEpilogue)
+ PreferPredicateOverEpilogue != PreferPredicateTy::FoldEpilogueTail)
return false;
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
- "disallowed while epilogue predication allowed!\n");
- LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ "disallowed while epilogue tail-folding allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue tail-folding\n");
return false;
}
// If scalar epilogue is explicitly required, we can't apply TF.
if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
"scalar epilogue is required\n");
return false;
}
@@ -8347,21 +8345,10 @@ static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
return false;
}
- // TF is not supported for some kinds of reductions that result in more
- // overhead.
- bool HasReductions = !LVL.getReductionVars().empty();
- bool HasSelectCmpReductions =
- HasReductions &&
- any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
- const RecurrenceDescriptor &RdxDesc = Reduction.second;
- RecurKind RK = RdxDesc.getRecurrenceKind();
- return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
- RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
- RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
- });
- if (HasSelectCmpReductions) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
- "select-cmp Reductions\n");
+ // TF is not supported for reductions right now.
+ if (!LVL.getReductionVars().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding is not supported for "
+ "reductions\n");
return false;
}
@@ -9253,7 +9240,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (isEpilogueTailFoldingAllowed(CM, LVL)) {
// TODO: Apply tail folding on the vectorized epilogue loop.
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
+ reportVectorizationInfo("FoldEpilogueTail flag is not supported yet, we "
+ "are falling back on vectorizing with scalar tail",
+ "EpilogueTailFoldingNotSupported", ORE, L);
}
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index e717b59935520..0c094fcd9fb42 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,20 +1,21 @@
; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
+; RUN: -prefer-predicate-over-epilogue=fold-epilogue-tail 2>&1 | FileCheck %s
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: --disable-output -prefer-predicate-over-epilogue=fold-epilogue-tail 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
-; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: --disable-output -prefer-predicate-over-epilogue=fold-epilogue-tail 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
define void @test_epilogue_tf(ptr %a, i64 %n) {
; CHECK: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: Epilogue tail folding is enabled
+; CHECK: LV: epilogue tail-folding is not supported yet
+; CHECK: LV: FoldEpilogueTail flag is not supported yet, we are falling back on vectorizing with scalar tail.
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
@@ -32,8 +33,8 @@ for.end:
define void @epilogue_is_disabled(ptr %a, i64 %n) {
; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
-; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
-; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue tail-folding
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
@@ -53,7 +54,7 @@ for.end:
@CD = common global [1024 x i32] zeroinitializer, align 4
define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
-; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+; CHECK-INTERLEAVE: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %for.body
@@ -80,7 +81,7 @@ for.end: ; preds = %for.body
define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
-; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %loop.header
loop.header:
@@ -121,7 +122,7 @@ for.end:
define i32 @max_reduction_epilog(ptr %src, i64 %N) {
; CHECK: LV: Checking a loop in 'max_reduction_epilog'
-; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+; CHECK: LV: Epilogue tail-folding is not supported for reductions
entry:
br label %loop
>From e3c1f19c36fdcef54681f914b9de11d2000e44da Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 16 Apr 2026 12:03:39 +0000
Subject: [PATCH 5/5] rebase and restore original format to minimize diffs.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 31 ++++++++++---------
1 file changed, 16 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 53f00de0e30e8..4ef9e4a2464f9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -221,23 +221,24 @@ namespace PreferPredicateTy {
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"prefer-predicate-over-epilogue",
- cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,
+ cl::init(PreferPredicateTy::ScalarEpilogue),
+ cl::Hidden,
cl::desc("Tail-folding and predication preferences over creating a scalar "
"epilogue loop."),
- cl::values(
- clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue",
- "Don't tail-predicate loops, create scalar epilogue"),
- clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
- "predicate-else-scalar-epilogue",
- "prefer tail-folding, create scalar epilogue if tail "
- "folding fails."),
- clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
- "predicate-dont-vectorize",
- "prefers tail-folding, don't attempt vectorization if "
- "tail-folding fails."),
- clEnumValN(PreferPredicateTy::FoldEpilogueTail, "fold-epilogue-tail",
- "prefers tail-folded vector epilogue, falling back on "
- "an epilogue if it fails.")));
+ cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
+ "scalar-epilogue",
+ "Don't tail-predicate loops, create scalar epilogue"),
+ clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
+ "predicate-else-scalar-epilogue",
+ "prefer tail-folding, create scalar epilogue if tail "
+ "folding fails."),
+ clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
+ "predicate-dont-vectorize",
+ "prefers tail-folding, don't attempt vectorization if "
+ "tail-folding fails."),
+ clEnumValN(PreferPredicateTy::FoldEpilogueTail, "fold-epilogue-tail",
+ "prefers tail-folded vector epilogue, falling back on "
+ "an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
More information about the llvm-commits
mailing list