[llvm] [LV] Add -predicated-epilogue option for tail-folded epilogue (PR #190697)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 06:48:52 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/190697
>From 3d698afc8b741bc6188f94e42e179674eb1d56d9 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 6 Apr 2026 19:04:29 +0000
Subject: [PATCH 1/5] [LV] Add -predicated-epilogue option for tail-folded
epilogue
This is the very beginning of implementing tail folding on
epilogue loops where we can have vectorized unpredicated main loop
along with vectorized predicated epilogue loop.
This patch is doing:
- Add a new value -predicated-epilogue to the opt-in option
-prefer-predicate-over-epilogue
- Add validation logic to filter out the cases where tail folding
will not be applied
- Add some logs
- Add tests for the excluded cases
This is just the foundational work: introducing the option and the
excluded cases. The actual implementation of tail folding on
epilogue loops will come gradually in future patches.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 110 +++++++++++---
.../LoopVectorize/no-epilog-tail-folding.ll | 140 ++++++++++++++++++
2 files changed, 232 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e20ef0296daea..7fd0cd102de61 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -211,30 +211,35 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
// and predicate the instructions accordingly. If tail-folding fails, there are
// different fallback strategies depending on these values:
namespace PreferPredicateTy {
- enum Option {
- ScalarEpilogue = 0,
- PredicateElseScalarEpilogue,
- PredicateOrDontVectorize
- };
+// TODO: Rename enum values and flag names to use 'TailFold' instead of
+// 'Predicate' for clarity.
+enum Option {
+ ScalarEpilogue = 0,
+ PredicateElseScalarEpilogue,
+ PredicateOrDontVectorize,
+ PredicatedEpilogue
+};
} // namespace PreferPredicateTy
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"prefer-predicate-over-epilogue",
- cl::init(PreferPredicateTy::ScalarEpilogue),
- cl::Hidden,
+ cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,
cl::desc("Tail-folding and predication preferences over creating a scalar "
"epilogue loop."),
- cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
- "scalar-epilogue",
- "Don't tail-predicate loops, create scalar epilogue"),
- clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
- "predicate-else-scalar-epilogue",
- "prefer tail-folding, create scalar epilogue if tail "
- "folding fails."),
- clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
- "predicate-dont-vectorize",
- "prefers tail-folding, don't attempt vectorization if "
- "tail-folding fails.")));
+ cl::values(
+ clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue",
+ "Don't tail-predicate loops, create scalar epilogue"),
+ clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
+ "predicate-else-scalar-epilogue",
+ "prefer tail-folding, create scalar epilogue if tail "
+ "folding fails."),
+ clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
+ "predicate-dont-vectorize",
+ "prefers tail-folding, don't attempt vectorization if "
+ "tail-folding fails."),
+ clEnumValN(PreferPredicateTy::PredicatedEpilogue, "predicated-epilogue",
+ "prefers predicated vector epilogue, falling back on "
+ "scalar epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -853,6 +858,11 @@ enum EpilogueLowering {
// Loop hint indicating an epilogue is undesired, apply tail folding.
CM_EpilogueNotNeededFoldTail,
+ // Predicated vector epilogue requested; the scalar epilogue of the
+ // vectorized epilogue loop will be tail-folded if possible, otherwise
+ // a scalar epilogue is kept.
+ CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
+
// Directive indicating we must either fold the epilogue/tail or not vectorize
CM_EpilogueNotAllowedFoldTail
};
@@ -3514,6 +3524,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return FixedScalableVFPair::getNone();
break;
+ default:
+ // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ break;
}
// Now try the tail folding
@@ -8259,6 +8272,9 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) tail-folding compiler options, 3) loop
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
+// This function determines scalar epilogue lowering for the main vector loop
+// while scalar epilogue lowering for the predicated epilogue path is handled
+// separately in isEpilogueTailFoldingAllowed().
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
@@ -8279,6 +8295,8 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueNotNeededFoldTail;
case PreferPredicateTy::PredicateOrDontVectorize:
return CM_EpilogueNotAllowedFoldTail;
+ default:
+ break;
};
}
@@ -8298,6 +8316,58 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
+/// Check if we can apply tail-folding on the scalar loop of the vectorized
+/// epilogue loop, so that we can have vectorized unpredicated main loop along
+/// with vectorized predicated epilogue loop.
+static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
+ const LoopVectorizationLegality &LVL) {
+ // Epilogue TF is only enabled when explicitly requested via command line.
+ if (!PreferPredicateOverEpilogue.getNumOccurrences() ||
+ PreferPredicateOverEpilogue != PreferPredicateTy::PredicatedEpilogue)
+ return false;
+
+ if (!EnableEpilogueVectorization) {
+ LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
+ "disallowed while epilogue predication allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ return false;
+ }
+
+ // If scalar epilogue is explicitly required, we can't apply TF.
+ if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ "scalar epilogue is required\n");
+ return false;
+ }
+
+ // If having epilogue is NOT allowed, then no epilogue to apply TF for.
+ if (!CM.isEpilogueAllowed()) {
+ LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail folding for.\n");
+ return false;
+ }
+
+ // TF is not supported for some kinds of reductions that results in more
+ // overhead
+ bool HasReductions = !LVL.getReductionVars().empty();
+ bool HasSelectCmpReductions =
+ HasReductions &&
+ any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
+ const RecurrenceDescriptor &RdxDesc = Reduction.second;
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
+ });
+ if (HasSelectCmpReductions) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
+ "select-cmp Reductions\n");
+ return false;
+ }
+
+ // We can apply TF on the scalar loop of the vectorized epilogue.
+ return true;
+}
+
// Process the loop in the VPlan-native vectorization path. This path builds
// VPlan upfront in the vectorization pipeline, which allows to apply
// VPlan-to-VPlan transformations from the very beginning without modifying the
@@ -9180,6 +9250,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
ORE);
+ if (isEpilogueTailFoldingAllowed(CM, LVL)) {
+ // TODO: Apply tail folding on the vectorized epilogue loop.
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ }
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
new file mode 100644
index 0000000000000..7db85bb2f739e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -0,0 +1,140 @@
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
+; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
+; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+define void @test_epilogue_tf(ptr %a, i64 %n) {
+; CHECK: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK: LV: Epilogue tail folding is enabled
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+define void @epilogue_is_disabled(ptr %a, i64 %n) {
+; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+ at AB = common global [1024 x i32] zeroinitializer, align 4
+ at CD = common global [1024 x i32] zeroinitializer, align 4
+define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
+; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
+; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv
+ %tmp = load i32, ptr %arrayidx0, align 4
+ %tmp1 = or disjoint i64 %indvars.iv, 1
+ %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp1
+ %tmp2 = load i32, ptr %arrayidx1, align 4
+ %add = add nsw i32 %tmp, %C
+ %mul = mul nsw i32 %tmp2, %D
+ %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx2, align 4
+ %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp1
+ store i32 %mul, ptr %arrayidx3, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+ %cmp = icmp slt i64 %indvars.iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body
+ ret void
+}
+
+define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
+; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
+; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %loop.header
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i64 0, ptr %gep
+ br i1 true, label %loop.then, label %exit.2
+loop.then:
+ %cmp3 = icmp ne i64 %iv, %x
+ br i1 %cmp3, label %loop.latch, label %exit.1
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ br label %loop.header
+exit.1:
+ ret i16 0
+exit.2:
+ ret i16 1
+}
+
+define i32 @opt_for_size(ptr %p, i32 %n) optsize {
+; CHECK: LV: Checking a loop in 'opt_for_size'
+; CHECK: LV: No epilogue to apply tail folding for.
+entry:
+ br label %for.body
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
+ %0 = load i32, ptr %arrayidx, align 1
+ %cmp1 = icmp eq i32 %0, 0
+ %sel = select i1 %cmp1, i32 2, i32 1
+ store i32 %sel, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.end, label %for.body
+for.end:
+ ret i32 0
+}
+
+define i32 @max_reduction_epilog(ptr %src, i64 %N) {
+; CHECK: LV: Checking a loop in 'max_reduction_epilog'
+; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %icmp = icmp ugt i32 %load, %red
+ %select = select i1 %icmp, i32 %load, i32 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %N
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %select
+}
>From 4a9e587e42d774f2c053d1559a9144f5bb5930d6 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 01:08:38 +0000
Subject: [PATCH 2/5] Add 'REQUIRES: asserts' to the test file.
---
llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index 7db85bb2f739e..e717b59935520 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,3 +1,4 @@
+; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
>From 6d2f3a7e0db6fa1f7e42a189ef6b3a4c23dce697 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 14:47:25 +0000
Subject: [PATCH 3/5] improve readability and rebase
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7fd0cd102de61..7c81d85fdea8b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8346,8 +8346,8 @@ static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
return false;
}
- // TF is not supported for some kinds of reductions that results in more
- // overhead
+ // TF is not supported for some kinds of reductions that result in more
+ // overhead.
bool HasReductions = !LVL.getReductionVars().empty();
bool HasSelectCmpReductions =
HasReductions &&
>From 4786e0af16da045aeac7fea9982ae7928d55d4be Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 13 Apr 2026 21:04:46 +0000
Subject: [PATCH 4/5] resolve review comments - improve readability
---
.../Transforms/Vectorize/LoopVectorize.cpp | 70 ++++++++-----------
.../LoopVectorize/no-epilog-tail-folding.ll | 19 ++---
2 files changed, 40 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7c81d85fdea8b..b8d6c1781b243 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -211,14 +211,12 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
// and predicate the instructions accordingly. If tail-folding fails, there are
// different fallback strategies depending on these values:
namespace PreferPredicateTy {
-// TODO: Rename enum values and flag names to use 'TailFold' instead of
-// 'Predicate' for clarity.
-enum Option {
- ScalarEpilogue = 0,
- PredicateElseScalarEpilogue,
- PredicateOrDontVectorize,
- PredicatedEpilogue
-};
+ enum Option {
+ ScalarEpilogue = 0,
+ PredicateElseScalarEpilogue,
+ PredicateOrDontVectorize,
+ FoldEpilogueTail
+ };
} // namespace PreferPredicateTy
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
@@ -237,9 +235,9 @@ static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"predicate-dont-vectorize",
"prefers tail-folding, don't attempt vectorization if "
"tail-folding fails."),
- clEnumValN(PreferPredicateTy::PredicatedEpilogue, "predicated-epilogue",
- "prefers predicated vector epilogue, falling back on "
- "scalar epilogue if it fails.")));
+ clEnumValN(PreferPredicateTy::FoldEpilogueTail, "fold-epilogue-tail",
+ "prefers tail-folded vector epilogue, falling back on "
+ "an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -858,10 +856,10 @@ enum EpilogueLowering {
// Loop hint indicating an epilogue is undesired, apply tail folding.
CM_EpilogueNotNeededFoldTail,
- // Predicated vector epilogue requested; the scalar epilogue of the
- // vectorized epilogue loop will be tail-folded if possible, otherwise
- // a scalar epilogue is kept.
- CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
+ // Tail-folded vector epilogue requested; the scalar tail will be folded into
+ // the vectorized epilogue loop if possible, otherwise fall back on a scalar
+ // epilogue.
+ CM_EpilogueNotNeededFoldEpilogueTail,
// Directive indicating we must either fold the epilogue/tail or not vectorize
CM_EpilogueNotAllowedFoldTail
@@ -3525,7 +3523,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
break;
default:
- // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ // TODO: handle the case for CM_EpilogueNotNeededFoldEpilogueTail
break;
}
@@ -8273,7 +8271,7 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
// This function determines scalar epilogue lowering for the main vector loop
-// while scalar epilogue lowering for the predicated epilogue path is handled
+// while scalar epilogue lowering for the tail-folded epilogue path is handled
// separately in isEpilogueTailFoldingAllowed().
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
@@ -8316,26 +8314,26 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
-/// Check if we can apply tail-folding on the scalar loop of the vectorized
-/// epilogue loop, so that we can have vectorized unpredicated main loop along
-/// with vectorized predicated epilogue loop.
+/// Check if we can apply tail folding to the vectorized epilogue loop,
+/// enabling an unpredicated main vector loop with a tail-folded epilogue
+/// vector loop.
static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
const LoopVectorizationLegality &LVL) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!PreferPredicateOverEpilogue.getNumOccurrences() ||
- PreferPredicateOverEpilogue != PreferPredicateTy::PredicatedEpilogue)
+ PreferPredicateOverEpilogue != PreferPredicateTy::FoldEpilogueTail)
return false;
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
- "disallowed while epilogue predication allowed!\n");
- LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ "disallowed while epilogue tail-folding allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue tail-folding\n");
return false;
}
// If scalar epilogue is explicitly required, we can't apply TF.
if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
"scalar epilogue is required\n");
return false;
}
@@ -8346,21 +8344,10 @@ static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
return false;
}
- // TF is not supported for some kinds of reductions that result in more
- // overhead.
- bool HasReductions = !LVL.getReductionVars().empty();
- bool HasSelectCmpReductions =
- HasReductions &&
- any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
- const RecurrenceDescriptor &RdxDesc = Reduction.second;
- RecurKind RK = RdxDesc.getRecurrenceKind();
- return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
- RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
- RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
- });
- if (HasSelectCmpReductions) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
- "select-cmp Reductions\n");
+ // TF is not supported for reductions right now.
+ if (!LVL.getReductionVars().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding is not supported for "
+ "reductions\n");
return false;
}
@@ -9252,7 +9239,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (isEpilogueTailFoldingAllowed(CM, LVL)) {
// TODO: Apply tail folding on the vectorized epilogue loop.
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
+ reportVectorizationInfo("FoldEpilogueTail flag is not supported yet, we "
+ "are falling back on vectorizing with scalar tail",
+ "EpilogueTailFoldingNotSupported", ORE, L);
}
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index e717b59935520..0c094fcd9fb42 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,20 +1,21 @@
; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 | FileCheck %s
+; RUN: -prefer-predicate-over-epilogue=fold-epilogue-tail 2>&1 | FileCheck %s
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: --disable-output -prefer-predicate-over-epilogue=fold-epilogue-tail 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
-; RUN: --disable-output -prefer-predicate-over-epilogue=predicated-epilogue 2>&1 \
+; RUN: --disable-output -prefer-predicate-over-epilogue=fold-epilogue-tail 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
define void @test_epilogue_tf(ptr %a, i64 %n) {
; CHECK: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: Epilogue tail folding is enabled
+; CHECK: LV: epilogue tail-folding is not supported yet
+; CHECK: LV: FoldEpilogueTail flag is not supported yet, we are falling back on vectorizing with scalar tail.
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
@@ -32,8 +33,8 @@ for.end:
define void @epilogue_is_disabled(ptr %a, i64 %n) {
; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
-; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
-; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue tail-folding
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
@@ -53,7 +54,7 @@ for.end:
@CD = common global [1024 x i32] zeroinitializer, align 4
define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
-; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+; CHECK-INTERLEAVE: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %for.body
@@ -80,7 +81,7 @@ for.end: ; preds = %for.body
define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
-; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %loop.header
loop.header:
@@ -121,7 +122,7 @@ for.end:
define i32 @max_reduction_epilog(ptr %src, i64 %N) {
; CHECK: LV: Checking a loop in 'max_reduction_epilog'
-; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+; CHECK: LV: Epilogue tail-folding is not supported for reductions
entry:
br label %loop
>From 0bf70905d59bd4de9ab0196b5f17ae1b608489c7 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 16 Apr 2026 12:03:39 +0000
Subject: [PATCH 5/5] rebase and restore original format to minimize diffs.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 37 ++++++++++---------
1 file changed, 19 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b8d6c1781b243..168a9faeca865 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -221,23 +221,24 @@ namespace PreferPredicateTy {
static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
"prefer-predicate-over-epilogue",
- cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,
+ cl::init(PreferPredicateTy::ScalarEpilogue),
+ cl::Hidden,
cl::desc("Tail-folding and predication preferences over creating a scalar "
"epilogue loop."),
- cl::values(
- clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue",
- "Don't tail-predicate loops, create scalar epilogue"),
- clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
- "predicate-else-scalar-epilogue",
- "prefer tail-folding, create scalar epilogue if tail "
- "folding fails."),
- clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
- "predicate-dont-vectorize",
- "prefers tail-folding, don't attempt vectorization if "
- "tail-folding fails."),
- clEnumValN(PreferPredicateTy::FoldEpilogueTail, "fold-epilogue-tail",
- "prefers tail-folded vector epilogue, falling back on "
- "an epilogue if it fails.")));
+ cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
+ "scalar-epilogue",
+ "Don't tail-predicate loops, create scalar epilogue"),
+ clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
+ "predicate-else-scalar-epilogue",
+ "prefer tail-folding, create scalar epilogue if tail "
+ "folding fails."),
+ clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
+ "predicate-dont-vectorize",
+ "prefers tail-folding, don't attempt vectorization if "
+ "tail-folding fails."),
+ clEnumValN(PreferPredicateTy::FoldEpilogueTail, "fold-epilogue-tail",
+ "prefers tail-folded vector epilogue, falling back on "
+ "an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -8270,9 +8271,9 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) tail-folding compiler options, 3) loop
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
-// This function determines scalar epilogue lowering for the main vector loop
-// while scalar epilogue lowering for the tail-folded epilogue path is handled
-// separately in isEpilogueTailFoldingAllowed().
+// This function determines epilogue lowering for the main vector loop while
+// epilogue lowering for the tail-folded epilogue path is handled separately in
+// isEpilogueTailFoldingAllowed().
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
More information about the llvm-commits
mailing list