[llvm] [LV] Add -fold-epilogue-tail option for tail-folded epilogue (PR #190697)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 06:22:47 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/190697
>From eb025ae11a62d11c1c82f3d6bec7d9253d81a58b Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 6 Apr 2026 19:04:29 +0000
Subject: [PATCH 01/11] [LV] Add -predicated-epilogue option for tail-folded
epilogue
This is the very beginning of implementing tail folding on
epilogue loops where we can have vectorized unpredicated main loop
along with vectorized predicated epilogue loop.
This patch is doing:
- Add a new value -predicated-epilogue to the opt-in option
-prefer-predicate-over-epilogue
- Add validation logic to filter out the cases where tail folding
will not be applied
- Add some logs
- Add tests for the excluded cases
This is just the foundational work: introducing the option and the
excluded cases. The actual implementation of tail folding on
epilogue loops will come gradually in future patches.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 76 +++++++++-
.../LoopVectorize/no-epilog-tail-folding.ll | 140 ++++++++++++++++++
2 files changed, 214 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9dd4ab50a2938..980513fb77b16 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -203,7 +203,7 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
/// will try to fold the tail-loop (epilogue) into the vector body and predicate
/// the instructions accordingly. If tail-folding fails, there are different
/// fallback strategies depending on these values:
-enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail };
+enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail, PredicatedEpilogue };
static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
"tail-folding-policy", cl::init(TailFoldingPolicyTy::None), cl::Hidden,
@@ -216,7 +216,10 @@ static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
"appropriate."),
clEnumValN(TailFoldingPolicyTy::MustFoldTail, "must-fold-tail",
"always tail-fold, don't attempt vectorization if "
- "tail-folding fails.")));
+ "tail-folding fails."),
+ clEnumValN(TailFoldingPolicyTy::PredicatedEpilogue, "predicated-epilogue",
+ "prefers predicated vector epilogue, falling back on "
+ "scalar epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -804,6 +807,11 @@ enum EpilogueLowering {
// Loop hint indicating an epilogue is undesired, apply tail folding.
CM_EpilogueNotNeededFoldTail,
+ // Predicated vector epilogue requested; the scalar epilogue of the
+ // vectorized epilogue loop will be tail-folded if possible, otherwise
+ // a scalar epilogue is kept.
+ CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
+
// Directive indicating we must either fold the epilogue/tail or not vectorize
CM_EpilogueNotAllowedFoldTail
};
@@ -3047,6 +3055,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return FixedScalableVFPair::getNone();
break;
+ default:
+ // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ break;
}
// Now try the tail folding
@@ -7518,6 +7529,9 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) tail-folding compiler options, 3) loop
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
+// This function determines scalar epilogue lowering for the main vector loop
+// while scalar epilogue lowering for the predicated epilogue path is handled
+// separately in isEpilogueTailFoldingAllowed().
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
@@ -7538,6 +7552,8 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueNotNeededFoldTail;
case TailFoldingPolicyTy::MustFoldTail:
return CM_EpilogueNotAllowedFoldTail;
+ default:
+ break;
};
}
@@ -7557,6 +7573,58 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
+/// Check if we can apply tail-folding on the scalar loop of the vectorized
+/// epilogue loop, so that we can have vectorized unpredicated main loop along
+/// with vectorized predicated epilogue loop.
+static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
+ const LoopVectorizationLegality &LVL) {
+ // Epilogue TF is only enabled when explicitly requested via command line.
+ if (!TailFoldingPolicy.getNumOccurrences() ||
+ TailFoldingPolicy != TailFoldingPolicyTy::PredicatedEpilogue)
+ return false;
+
+ if (!EnableEpilogueVectorization) {
+ LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
+ "disallowed while epilogue predication allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ return false;
+ }
+
+ // If scalar epilogue is explicitly required, we can't apply TF.
+ if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ "scalar epilogue is required\n");
+ return false;
+ }
+
+ // If having epilogue is NOT allowed, then no epilogue to apply TF for.
+ if (!CM.isEpilogueAllowed()) {
+ LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail folding for.\n");
+ return false;
+ }
+
+ // TF is not supported for some kinds of reductions that results in more
+ // overhead
+ bool HasReductions = !LVL.getReductionVars().empty();
+ bool HasSelectCmpReductions =
+ HasReductions &&
+ any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
+ const RecurrenceDescriptor &RdxDesc = Reduction.second;
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
+ });
+ if (HasSelectCmpReductions) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
+ "select-cmp Reductions\n");
+ return false;
+ }
+
+ // We can apply TF on the scalar loop of the vectorized epilogue.
+ return true;
+}
+
// Process the loop in the VPlan-native vectorization path. This path builds
// VPlan upfront in the vectorization pipeline, which allows to apply
// VPlan-to-VPlan transformations from the very beginning without modifying the
@@ -8420,6 +8488,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
Hints, ORE);
+ if (isEpilogueTailFoldingAllowed(CM, LVL)) {
+ // TODO: Apply tail folding on the vectorized epilogue loop.
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ }
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
new file mode 100644
index 0000000000000..87d4d717b81c0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -0,0 +1,140 @@
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; RUN: -tail-folding-policy=predicated-epilogue 2>&1 | FileCheck %s
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
+; RUN: --disable-output -tail-folding-policy=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
+; RUN: --disable-output -tail-folding-policy=predicated-epilogue 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+define void @test_epilogue_tf(ptr %a, i64 %n) {
+; CHECK: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK: LV: Epilogue tail folding is enabled
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+define void @epilogue_is_disabled(ptr %a, i64 %n) {
+; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+ at AB = common global [1024 x i32] zeroinitializer, align 4
+ at CD = common global [1024 x i32] zeroinitializer, align 4
+define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
+; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
+; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv
+ %tmp = load i32, ptr %arrayidx0, align 4
+ %tmp1 = or disjoint i64 %indvars.iv, 1
+ %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp1
+ %tmp2 = load i32, ptr %arrayidx1, align 4
+ %add = add nsw i32 %tmp, %C
+ %mul = mul nsw i32 %tmp2, %D
+ %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx2, align 4
+ %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp1
+ store i32 %mul, ptr %arrayidx3, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+ %cmp = icmp slt i64 %indvars.iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body
+ ret void
+}
+
+define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
+; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
+; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+entry:
+ br label %loop.header
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i64 0, ptr %gep
+ br i1 true, label %loop.then, label %exit.2
+loop.then:
+ %cmp3 = icmp ne i64 %iv, %x
+ br i1 %cmp3, label %loop.latch, label %exit.1
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ br label %loop.header
+exit.1:
+ ret i16 0
+exit.2:
+ ret i16 1
+}
+
+define i32 @opt_for_size(ptr %p, i32 %n) optsize {
+; CHECK: LV: Checking a loop in 'opt_for_size'
+; CHECK: LV: No epilogue to apply tail folding for.
+entry:
+ br label %for.body
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
+ %0 = load i32, ptr %arrayidx, align 1
+ %cmp1 = icmp eq i32 %0, 0
+ %sel = select i1 %cmp1, i32 2, i32 1
+ store i32 %sel, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.end, label %for.body
+for.end:
+ ret i32 0
+}
+
+define i32 @max_reduction_epilog(ptr %src, i64 %N) {
+; CHECK: LV: Checking a loop in 'max_reduction_epilog'
+; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %iv
+ %load = load i32, ptr %gep, align 1
+ %icmp = icmp ugt i32 %load, %red
+ %select = select i1 %icmp, i32 %load, i32 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %N
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %select
+}
>From b3429abf5e3ef7b488b0c8ca4b3c47653d5e1d24 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 01:08:38 +0000
Subject: [PATCH 02/11] Add 'REQUIRES: asserts' to the test file.
---
llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index 87d4d717b81c0..8ce47d8a963f0 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,3 +1,4 @@
+; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -tail-folding-policy=predicated-epilogue 2>&1 | FileCheck %s
>From c383980b13d095b34d705f774f4029ebc082ab14 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 7 Apr 2026 14:47:25 +0000
Subject: [PATCH 03/11] improve readability and rebase
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 980513fb77b16..148a5dbeed783 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7603,8 +7603,8 @@ static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
return false;
}
- // TF is not supported for some kinds of reductions that results in more
- // overhead
+ // TF is not supported for some kinds of reductions that result in more
+ // overhead.
bool HasReductions = !LVL.getReductionVars().empty();
bool HasSelectCmpReductions =
HasReductions &&
>From c6cf640996c8823ceb0d4d6ac574ed428f43115d Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 13 Apr 2026 21:04:46 +0000
Subject: [PATCH 04/11] resolve review comments - improve readability
---
.../Transforms/Vectorize/LoopVectorize.cpp | 58 ++++++++-----------
.../LoopVectorize/no-epilog-tail-folding.ll | 19 +++---
2 files changed, 35 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 148a5dbeed783..bc0a006a15fcf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -203,7 +203,7 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
/// will try to fold the tail-loop (epilogue) into the vector body and predicate
/// the instructions accordingly. If tail-folding fails, there are different
/// fallback strategies depending on these values:
-enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail, PredicatedEpilogue };
+enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail, FoldEpilogueTail };
static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
"tail-folding-policy", cl::init(TailFoldingPolicyTy::None), cl::Hidden,
@@ -217,9 +217,9 @@ static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
clEnumValN(TailFoldingPolicyTy::MustFoldTail, "must-fold-tail",
"always tail-fold, don't attempt vectorization if "
"tail-folding fails."),
- clEnumValN(TailFoldingPolicyTy::PredicatedEpilogue, "predicated-epilogue",
- "prefers predicated vector epilogue, falling back on "
- "scalar epilogue if it fails.")));
+ clEnumValN(TailFoldingPolicyTy::FoldEpilogueTail, "fold-epilogue-tail",
+ "prefers tail-folded vector epilogue, falling back on "
+ "an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -807,10 +807,10 @@ enum EpilogueLowering {
// Loop hint indicating an epilogue is undesired, apply tail folding.
CM_EpilogueNotNeededFoldTail,
- // Predicated vector epilogue requested; the scalar epilogue of the
- // vectorized epilogue loop will be tail-folded if possible, otherwise
- // a scalar epilogue is kept.
- CM_ScalarEpilogueNotNeededUsePredicatedEpilogue,
+ // Tail-folded vector epilogue requested; the scalar tail will be folded into
+ // the vectorized epilogue loop if possible, otherwise fall back on a scalar
+ // epilogue.
+ CM_EpilogueNotNeededFoldEpilogueTail,
// Directive indicating we must either fold the epilogue/tail or not vectorize
CM_EpilogueNotAllowedFoldTail
@@ -3056,7 +3056,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
break;
default:
- // TODO: handle the case for CM_ScalarEpilogueNotNeededUsePredicatedEpilogue
+ // TODO: handle the case for CM_EpilogueNotNeededFoldEpilogueTail
break;
}
@@ -7530,7 +7530,7 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
// This function determines scalar epilogue lowering for the main vector loop
-// while scalar epilogue lowering for the predicated epilogue path is handled
+// while scalar epilogue lowering for the tail-folded epilogue path is handled
// separately in isEpilogueTailFoldingAllowed().
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
@@ -7573,26 +7573,26 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
-/// Check if we can apply tail-folding on the scalar loop of the vectorized
-/// epilogue loop, so that we can have vectorized unpredicated main loop along
-/// with vectorized predicated epilogue loop.
+/// Check if we can apply tail folding to the vectorized epilogue loop,
+/// enabling an unpredicated main vector loop with a tail-folded epilogue
+/// vector loop.
static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
const LoopVectorizationLegality &LVL) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!TailFoldingPolicy.getNumOccurrences() ||
- TailFoldingPolicy != TailFoldingPolicyTy::PredicatedEpilogue)
+ TailFoldingPolicy != TailFoldingPolicyTy::FoldEpilogueTail)
return false;
if (!EnableEpilogueVectorization) {
LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
- "disallowed while epilogue predication allowed!\n");
- LLVM_DEBUG(dbgs() << "LV: Disallow epilogue predication\n");
+ "disallowed while epilogue tail-folding allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Disallow epilogue tail-folding\n");
return false;
}
// If scalar epilogue is explicitly required, we can't apply TF.
if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding can't be applied because "
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
"scalar epilogue is required\n");
return false;
}
@@ -7603,21 +7603,10 @@ static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
return false;
}
- // TF is not supported for some kinds of reductions that result in more
- // overhead.
- bool HasReductions = !LVL.getReductionVars().empty();
- bool HasSelectCmpReductions =
- HasReductions &&
- any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
- const RecurrenceDescriptor &RdxDesc = Reduction.second;
- RecurKind RK = RdxDesc.getRecurrenceKind();
- return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
- RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
- RecurrenceDescriptor::isMinMaxRecurrenceKind(RK);
- });
- if (HasSelectCmpReductions) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is not supported for "
- "select-cmp Reductions\n");
+ // TF is not supported for reductions right now.
+ if (!LVL.getReductionVars().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding is not supported for "
+ "reductions\n");
return false;
}
@@ -8490,7 +8479,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (isEpilogueTailFoldingAllowed(CM, LVL)) {
// TODO: Apply tail folding on the vectorized epilogue loop.
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail folding is enabled\n");
+ LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
+ reportVectorizationInfo("FoldEpilogueTail flag is not supported yet, we "
+ "are falling back on vectorizing with scalar tail",
+ "EpilogueTailFoldingNotSupported", ORE, L);
}
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index 8ce47d8a963f0..41b29650ae79d 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -1,20 +1,21 @@
; REQUIRES: asserts
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -tail-folding-policy=predicated-epilogue 2>&1 | FileCheck %s
+; RUN: -tail-folding-policy=fold-epilogue-tail 2>&1 | FileCheck %s
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -tail-folding-policy=predicated-epilogue 2>&1 \
+; RUN: --disable-output -tail-folding-policy=fold-epilogue-tail 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
-; RUN: --disable-output -tail-folding-policy=predicated-epilogue 2>&1 \
+; RUN: --disable-output -tail-folding-policy=fold-epilogue-tail 2>&1 \
; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
define void @test_epilogue_tf(ptr %a, i64 %n) {
; CHECK: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: Epilogue tail folding is enabled
+; CHECK: LV: epilogue tail-folding is not supported yet
+; CHECK: LV: FoldEpilogueTail flag is not supported yet, we are falling back on vectorizing with scalar tail.
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
@@ -32,8 +33,8 @@ for.end:
define void @epilogue_is_disabled(ptr %a, i64 %n) {
; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
-; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue predication allowed!
-; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue predication
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue tail-folding
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
@@ -53,7 +54,7 @@ for.end:
@CD = common global [1024 x i32] zeroinitializer, align 4
define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
-; CHECK-INTERLEAVE: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+; CHECK-INTERLEAVE: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %for.body
@@ -80,7 +81,7 @@ for.end: ; preds = %for.body
define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
-; CHECK: LV: Epilogue tail folding can't be applied because scalar epilogue is required
+; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %loop.header
loop.header:
@@ -121,7 +122,7 @@ for.end:
define i32 @max_reduction_epilog(ptr %src, i64 %N) {
; CHECK: LV: Checking a loop in 'max_reduction_epilog'
-; CHECK: LV: Epilogue tail folding is not supported for select-cmp Reductions
+; CHECK: LV: Epilogue tail-folding is not supported for reductions
entry:
br label %loop
>From e912cfc20b6a7e43aa491d84fed32cc06ae411f0 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 16 Apr 2026 12:03:39 +0000
Subject: [PATCH 05/11] rebase and format.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bc0a006a15fcf..7768ed42f1d3e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -218,7 +218,7 @@ static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
"always tail-fold, don't attempt vectorization if "
"tail-folding fails."),
clEnumValN(TailFoldingPolicyTy::FoldEpilogueTail, "fold-epilogue-tail",
- "prefers tail-folded vector epilogue, falling back on "
+ "prefer tail-folded vector epilogue, falling back on "
"an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
@@ -7529,9 +7529,9 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// for minimum code-size, 2) tail-folding compiler options, 3) loop
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
-// This function determines scalar epilogue lowering for the main vector loop
-// while scalar epilogue lowering for the tail-folded epilogue path is handled
-// separately in isEpilogueTailFoldingAllowed().
+// This function determines epilogue lowering for the main vector loop while
+// epilogue lowering for the tail-folded epilogue path is handled separately in
+// isEpilogueTailFoldingAllowed().
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
>From bd82c85ae7e8176a5b900c1a3ecd2a8b86943e05 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 21 Apr 2026 20:23:32 +0000
Subject: [PATCH 06/11] rebase and format
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7768ed42f1d3e..41ef6309a61d7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -203,7 +203,12 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
/// will try to fold the tail-loop (epilogue) into the vector body and predicate
/// the instructions accordingly. If tail-folding fails, there are different
/// fallback strategies depending on these values:
-enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail, FoldEpilogueTail };
+enum class TailFoldingPolicyTy {
+ None = 0,
+ PreferFoldTail,
+ MustFoldTail,
+ FoldEpilogueTail
+};
static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
"tail-folding-policy", cl::init(TailFoldingPolicyTy::None), cl::Hidden,
>From f945e755ef46ef7c85b82dc8cf2c0cf7a1700833 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Tue, 21 Apr 2026 20:48:59 +0000
Subject: [PATCH 07/11] remove isEpilogueTailFoldingAllowed()
---
.../Transforms/Vectorize/LoopVectorize.cpp | 44 +-----
.../LoopVectorize/no-epilog-tail-folding.ll | 136 ++----------------
2 files changed, 10 insertions(+), 170 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 41ef6309a61d7..0973cb27768ea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7578,47 +7578,6 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
-/// Check if we can apply tail folding to the vectorized epilogue loop,
-/// enabling an unpredicated main vector loop with a tail-folded epilogue
-/// vector loop.
-static bool isEpilogueTailFoldingAllowed(const LoopVectorizationCostModel &CM,
- const LoopVectorizationLegality &LVL) {
- // Epilogue TF is only enabled when explicitly requested via command line.
- if (!TailFoldingPolicy.getNumOccurrences() ||
- TailFoldingPolicy != TailFoldingPolicyTy::FoldEpilogueTail)
- return false;
-
- if (!EnableEpilogueVectorization) {
- LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
- "disallowed while epilogue tail-folding allowed!\n");
- LLVM_DEBUG(dbgs() << "LV: Disallow epilogue tail-folding\n");
- return false;
- }
-
- // If scalar epilogue is explicitly required, we can't apply TF.
- if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
- "scalar epilogue is required\n");
- return false;
- }
-
- // If having epilogue is NOT allowed, then no epilogue to apply TF for.
- if (!CM.isEpilogueAllowed()) {
- LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail folding for.\n");
- return false;
- }
-
- // TF is not supported for reductions right now.
- if (!LVL.getReductionVars().empty()) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding is not supported for "
- "reductions\n");
- return false;
- }
-
- // We can apply TF on the scalar loop of the vectorized epilogue.
- return true;
-}
-
// Process the loop in the VPlan-native vectorization path. This path builds
// VPlan upfront in the vectorization pipeline, which allows to apply
// VPlan-to-VPlan transformations from the very beginning without modifying the
@@ -8482,7 +8441,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
Hints, ORE);
- if (isEpilogueTailFoldingAllowed(CM, LVL)) {
+ if (TailFoldingPolicy.getNumOccurrences() &&
+ TailFoldingPolicy == TailFoldingPolicyTy::FoldEpilogueTail) {
// TODO: Apply tail folding on the vectorized epilogue loop.
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
reportVectorizationInfo("FoldEpilogueTail flag is not supported yet, we "
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
index 41b29650ae79d..a4896c92d661d 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
@@ -2,141 +2,21 @@
; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -tail-folding-policy=fold-epilogue-tail 2>&1 | FileCheck %s
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -tail-folding-policy=fold-epilogue-tail 2>&1 \
-; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
-
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-interleaved-mem-accesses=true \
-; RUN: --disable-output -tail-folding-policy=fold-epilogue-tail 2>&1 \
-; RUN: | FileCheck %s --check-prefix=CHECK-INTERLEAVE
-
-target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
-
-define void @test_epilogue_tf(ptr %a, i64 %n) {
+define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is not supported yet
; CHECK: LV: FoldEpilogueTail flag is not supported yet, we are falling back on vectorizing with scalar tail.
-entry:
- %cmp1 = icmp sgt i64 %n, 0
- br i1 %cmp1, label %for.body, label %for.end
-for.body:
- %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
- store i32 1, ptr %arrayidx, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp ne i64 %indvars.iv.next, %n
- br i1 %exitcond, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
-define void @epilogue_is_disabled(ptr %a, i64 %n) {
-; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
-; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
-; CHECK-DISABLED-EPILOG-NEXT: LV: Disallow epilogue tail-folding
-entry:
- %cmp1 = icmp sgt i64 %n, 0
- br i1 %cmp1, label %for.body, label %for.end
-for.body:
- %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
- store i32 1, ptr %arrayidx, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp ne i64 %indvars.iv.next, %n
- br i1 %exitcond, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
- at AB = common global [1024 x i32] zeroinitializer, align 4
- at CD = common global [1024 x i32] zeroinitializer, align 4
-define void @interleave_requires_scalar_epilog(i32 %C, i32 %D) {
-; CHECK-INTERLEAVE: LV: Checking a loop in 'interleave_requires_scalar_epilog'
-; CHECK-INTERLEAVE: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
entry:
br label %for.body
-for.body: ; preds = %for.body, %entry
- %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
- %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv
- %tmp = load i32, ptr %arrayidx0, align 4
- %tmp1 = or disjoint i64 %indvars.iv, 1
- %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp1
- %tmp2 = load i32, ptr %arrayidx1, align 4
- %add = add nsw i32 %tmp, %C
- %mul = mul nsw i32 %tmp2, %D
- %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv
- store i32 %add, ptr %arrayidx2, align 4
- %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp1
- store i32 %mul, ptr %arrayidx3, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
- %cmp = icmp slt i64 %indvars.iv.next, 1024
- br i1 %cmp, label %for.body, label %for.end
-
-for.end: ; preds = %for.body
- ret void
-}
-
-define i16 @early_exit_requires_scalar_epilog(ptr %dst, i64 %x) {
-; CHECK: LV: Checking a loop in 'early_exit_requires_scalar_epilog'
-; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
-entry:
- br label %loop.header
-loop.header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
- %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
- store i64 0, ptr %gep
- br i1 true, label %loop.then, label %exit.2
-loop.then:
- %cmp3 = icmp ne i64 %iv, %x
- br i1 %cmp3, label %loop.latch, label %exit.1
-loop.latch:
- %iv.next = add i64 %iv, 1
- br label %loop.header
-exit.1:
- ret i16 0
-exit.2:
- ret i16 1
-}
-
-define i32 @opt_for_size(ptr %p, i32 %n) optsize {
-; CHECK: LV: Checking a loop in 'opt_for_size'
-; CHECK: LV: No epilogue to apply tail folding for.
-entry:
- br label %for.body
for.body:
- %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
- %0 = load i32, ptr %arrayidx, align 1
- %cmp1 = icmp eq i32 %0, 0
- %sel = select i1 %cmp1, i32 2, i32 1
- store i32 %sel, ptr %arrayidx, align 1
- %inc = add nsw i32 %iv, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %for.end, label %for.body
-for.end:
- ret i32 0
-}
-
-define i32 @max_reduction_epilog(ptr %src, i64 %N) {
-; CHECK: LV: Checking a loop in 'max_reduction_epilog'
-; CHECK: LV: Epilogue tail-folding is not supported for reductions
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %red = phi i32 [ 0, %entry ], [ %select, %loop ]
- %gep = getelementptr inbounds i32, ptr %src, i64 %iv
- %load = load i32, ptr %gep, align 1
- %icmp = icmp ugt i32 %load, %red
- %select = select i1 %icmp, i32 %load, i32 %red
- %iv.next = add i64 %iv, 1
- %icmp3 = icmp eq i64 %iv, %N
- br i1 %icmp3, label %exit, label %loop
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
exit:
- ret i32 %select
+ ret void
}
>From 47ef4f166892025d654a25a826b45ac0ff77bd93 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 23 Apr 2026 13:08:53 +0000
Subject: [PATCH 08/11] resolve review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 43 +++++++++++--------
...og-tail-folding.ll => fold-epilog-tail.ll} | 2 +-
2 files changed, 27 insertions(+), 18 deletions(-)
rename llvm/test/Transforms/LoopVectorize/{no-epilog-tail-folding.ll => fold-epilog-tail.ll} (85%)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0973cb27768ea..de6f31bfc162f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,11 +198,18 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum allowed number of runtime memory checks"));
-/// Option tail-folding-policy indicates that an epilogue is undesired, that
-/// tail folding is preferred, and this lists all options. I.e., the vectorizer
-/// will try to fold the tail-loop (epilogue) into the vector body and predicate
-/// the instructions accordingly. If tail-folding fails, there are different
-/// fallback strategies depending on these values:
+/// Note: This currently only applies to `llvm.masked.load` and
+/// `llvm.masked.store`. TODO: Extend this to cover other operations as needed.
+static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
+ "force-target-supports-masked-memory-ops", cl::init(false), cl::Hidden,
+ cl::desc("Assume the target supports masked memory operations (used for "
+ "testing)."));
+
+/// Option tail-folding-policy controls the tail-folding strategy and lists all
+/// available options. The vectorizer will attempt to fold the tail-loop into
+/// the vector loop (main/epilogue loops) and predicate the instructions
+/// accordingly. If tail-folding fails, there are different fallback strategies
+/// depending on these values:
enum class TailFoldingPolicyTy {
None = 0,
PreferFoldTail,
@@ -223,7 +230,7 @@ static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
"always tail-fold, don't attempt vectorization if "
"tail-folding fails."),
clEnumValN(TailFoldingPolicyTy::FoldEpilogueTail, "fold-epilogue-tail",
- "prefer tail-folded vector epilogue, falling back on "
+ "prefer tail-folded vector epilogue, fall back to "
"an epilogue if it fails.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
@@ -813,8 +820,9 @@ enum EpilogueLowering {
CM_EpilogueNotNeededFoldTail,
// Tail-folded vector epilogue requested; the scalar tail will be folded into
- // the vectorized epilogue loop if possible, otherwise fall back on a scalar
- // epilogue.
+ // the vectorized epilogue loop if possible, otherwise fall back to an
+ // epilogue. This status is specifically for the CM instance of the
+ // tail-folded epilogue.
CM_EpilogueNotNeededFoldEpilogueTail,
// Directive indicating we must either fold the epilogue/tail or not vectorize
@@ -3061,7 +3069,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
break;
default:
- // TODO: handle the case for CM_EpilogueNotNeededFoldEpilogueTail
+ // Ignore the case of CM_EpilogueNotNeededFoldEpilogueTail for now.
+ // TODO: delete/clarify the comment when there is a handling for the case of
+ // CM_EpilogueNotNeededFoldEpilogueTail
break;
}
@@ -7535,8 +7545,8 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
// is suitable for tail-folding.
// This function determines epilogue lowering for the main vector loop while
-// epilogue lowering for the tail-folded epilogue path is handled separately in
-// isEpilogueTailFoldingAllowed().
+// epilogue lowering for the tail-folded epilogue path will be handled
+// separately.
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
@@ -8441,13 +8451,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
Hints, ORE);
- if (TailFoldingPolicy.getNumOccurrences() &&
- TailFoldingPolicy == TailFoldingPolicyTy::FoldEpilogueTail) {
- // TODO: Apply tail folding on the vectorized epilogue loop.
+ if (TailFoldingPolicy == TailFoldingPolicyTy::FoldEpilogueTail) {
+ // TODO: Apply tail-folding on the vectorized epilogue loop.
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
- reportVectorizationInfo("FoldEpilogueTail flag is not supported yet, we "
- "are falling back on vectorizing with scalar tail",
- "EpilogueTailFoldingNotSupported", ORE, L);
+ reportVectorizationInfo("The tail-folding policy fold-epilogue-tail is not "
+ "supported yet, fall back to an epilogue",
+ "UnsupportedTailFoldingPolicy", ORE, L);
}
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
diff --git a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll
similarity index 85%
rename from llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
rename to llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll
index a4896c92d661d..1c90b5222bbca 100644
--- a/llvm/test/Transforms/LoopVectorize/no-epilog-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll
@@ -5,7 +5,7 @@
define void @test_epilogue_tf(ptr %A, i64 %n) {
; CHECK: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is not supported yet
-; CHECK: LV: FoldEpilogueTail flag is not supported yet, we are falling back on vectorizing with scalar tail.
+; CHECK: The tail-folding policy fold-epilogue-tail is not supported yet, fall back to an epilogue.
entry:
br label %for.body
>From 0683026196fde87f89145111364d089ef4f285a4 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Sat, 25 Apr 2026 12:41:11 +0000
Subject: [PATCH 09/11] remove unintended change
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 -------
1 file changed, 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de6f31bfc162f..de0382854214a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,13 +198,6 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum allowed number of runtime memory checks"));
-/// Note: This currently only applies to `llvm.masked.load` and
-/// `llvm.masked.store`. TODO: Extend this to cover other operations as needed.
-static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
- "force-target-supports-masked-memory-ops", cl::init(false), cl::Hidden,
- cl::desc("Assume the target supports masked memory operations (used for "
- "testing)."));
-
/// Option tail-folding-policy controls the tail-folding strategy and lists all
/// available options. The vectorizer will attempt to fold the tail-loop into
/// the vector loop (main/epilogue loops) and predicate the instructions
>From c60ace2f512d41a55fd6a8bfd9ce4b7925830726 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 27 Apr 2026 17:36:15 +0000
Subject: [PATCH 10/11] create getEpilogueTailLowering to handle conflicts
between tail lowering of the vector epilogue and tail lowering of the vector
main loop
---
.../Transforms/Vectorize/LoopVectorize.cpp | 63 ++++++++++---
.../LoopVectorize/fold-epilog-tail.ll | 22 -----
.../LoopVectorize/fold-epilogue-tail.ll | 88 +++++++++++++++++++
3 files changed, 141 insertions(+), 32 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de0382854214a..ffb46955a399a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -207,7 +207,7 @@ enum class TailFoldingPolicyTy {
None = 0,
PreferFoldTail,
MustFoldTail,
- FoldEpilogueTail
+ PreferFoldEpilogueTail
};
static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
@@ -222,9 +222,11 @@ static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
clEnumValN(TailFoldingPolicyTy::MustFoldTail, "must-fold-tail",
"always tail-fold, don't attempt vectorization if "
"tail-folding fails."),
- clEnumValN(TailFoldingPolicyTy::FoldEpilogueTail, "fold-epilogue-tail",
- "prefer tail-folded vector epilogue, fall back to "
- "an epilogue if it fails.")));
+ clEnumValN(
+ TailFoldingPolicyTy::PreferFoldEpilogueTail,
+ "prefer-fold-epilogue-tail",
+ "prefer tail-folded vector epilogue, otherwise fallback to a "
+ "normal epilogue.")));
static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
"force-tail-folding-style", cl::desc("Force the tail folding style"),
@@ -813,7 +815,7 @@ enum EpilogueLowering {
CM_EpilogueNotNeededFoldTail,
// Tail-folded vector epilogue requested; the scalar tail will be folded into
- // the vectorized epilogue loop if possible, otherwise fall back to an
+ // the vectorized epilogue loop if possible, otherwise fall back to a normal
// epilogue. This status is specifically for the CM instance of the
// tail-folded epilogue.
CM_EpilogueNotNeededFoldEpilogueTail,
@@ -7539,7 +7541,7 @@ void LoopVectorizationPlanner::addMinimumIterationCheck(
// is suitable for tail-folding.
// This function determines epilogue lowering for the main vector loop while
// epilogue lowering for the tail-folded epilogue path will be handled
-// separately.
+// separately in getEpilogueTailLowering.
static EpilogueLowering
getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
bool OptForSize, TargetTransformInfo *TTI,
@@ -7581,6 +7583,43 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
return CM_EpilogueAllowed;
}
+/// Determine how to lower the epilogue for the vector epilogue loop.
+/// Check if there are any conflicts that prevent tail-folding the epilogue.
+/// \return CM_EpilogueNotNeededFoldEpilogueTail if epilogue tail-folding
+/// is possible, otherwise CM_EpilogueAllowed.
+static EpilogueLowering
+getEpilogueTailLowering(const LoopVectorizationCostModel &CM) {
+ // Epilogue TF is only enabled when explicitly requested via command line.
+ if (TailFoldingPolicy != TailFoldingPolicyTy::PreferFoldEpilogueTail)
+ // fallback to a normal epilogue.
+ return CM_EpilogueAllowed;
+
+ if (!EnableEpilogueVectorization) {
+ LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
+ "disallowed while epilogue tail-folding allowed!\n");
+ LLVM_DEBUG(dbgs() << "LV: Fallback to a normal epilogue\n");
+ return CM_EpilogueAllowed;
+ }
+
+ // If scalar epilogue is explicitly required, we can't apply TF.
+ if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
+ "scalar epilogue is required\n");
+ LLVM_DEBUG(dbgs() << "LV: Fallback to a normal epilogue\n");
+ return CM_EpilogueAllowed;
+ }
+
+ // If having epilogue is NOT allowed, then no epilogue to apply TF for.
+ if (!CM.isEpilogueAllowed()) {
+ LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n");
+ LLVM_DEBUG(dbgs() << "LV: Fallback to a normal epilogue\n");
+ return CM_EpilogueAllowed;
+ }
+
+ // We can apply tail-folding on the vectorized epilogue loop.
+ return CM_EpilogueNotNeededFoldEpilogueTail;
+}
+
// Process the loop in the VPlan-native vectorization path. This path builds
// VPlan upfront in the vectorization pipeline, which allows to apply
// VPlan-to-VPlan transformations from the very beginning without modifying the
@@ -8444,13 +8483,17 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
Hints, ORE);
- if (TailFoldingPolicy == TailFoldingPolicyTy::FoldEpilogueTail) {
+ EpilogueLowering EpilogueTailLoweringStatus = getEpilogueTailLowering(CM);
+ if (EpilogueTailLoweringStatus ==
+ EpilogueLowering::CM_EpilogueNotNeededFoldEpilogueTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
- reportVectorizationInfo("The tail-folding policy fold-epilogue-tail is not "
- "supported yet, fall back to an epilogue",
- "UnsupportedTailFoldingPolicy", ORE, L);
+ reportVectorizationInfo(
+ "The tail-folding policy prefer-fold-epilogue-tail is not supported "
+ "yet, fall back to a normal epilogue",
+ "UnsupportedTailFoldingPolicy", ORE, L);
}
+
// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll
deleted file mode 100644
index 1c90b5222bbca..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/fold-epilog-tail.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -tail-folding-policy=fold-epilogue-tail 2>&1 | FileCheck %s
-
-define void @test_epilogue_tf(ptr %A, i64 %n) {
-; CHECK: LV: Checking a loop in 'test_epilogue_tf'
-; CHECK: LV: epilogue tail-folding is not supported yet
-; CHECK: The tail-folding policy fold-epilogue-tail is not supported yet, fall back to an epilogue.
-entry:
- br label %for.body
-
-for.body:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
- store i8 1, ptr %arrayidx, align 1
- %iv.next = add nuw nsw i64 %iv, 1
- %exitcond = icmp ne i64 %iv.next, %n
- br i1 %exitcond, label %for.body, label %exit
-
-exit:
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
new file mode 100644
index 0000000000000..0279ce99143f4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -0,0 +1,88 @@
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; RUN: -tail-folding-policy=prefer-fold-epilogue-tail 2>&1 | FileCheck %s
+
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
+; RUN: --disable-output -tail-folding-policy=prefer-fold-epilogue-tail 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+define void @test_epilogue_tf(ptr %A, i64 %n) {
+; CHECK: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK: LV: epilogue tail-folding is not supported yet
+; CHECK: The tail-folding policy prefer-fold-epilogue-tail is not supported yet, fall back to a normal epilogue.
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ store i8 1, ptr %arrayidx, align 1
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, %n
+ br i1 %exitcond, label %for.body, label %exit
+
+exit:
+ ret void
+}
+
+define void @epilogue_is_disabled(ptr %a, i64 %n) {
+; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
+; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
+; CHECK-DISABLED-EPILOG-NEXT: LV: Fallback to a normal epilogue
+entry:
+ %cmp1 = icmp sgt i64 %n, 0
+ br i1 %cmp1, label %for.body, label %for.end
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+ store i32 1, ptr %arrayidx, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, %n
+ br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+define i16 @require_scalar_epilogue(ptr %dst, i64 %x) {
+; CHECK: LV: Checking a loop in 'require_scalar_epilogue'
+; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
+; CHECK-NEXT: LV: Fallback to a normal epilogue
+entry:
+ br label %loop.header
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
+ store i64 0, ptr %gep
+ br i1 true, label %loop.then, label %exit.2
+loop.then:
+ %cmp3 = icmp ne i64 %iv, %x
+ br i1 %cmp3, label %loop.latch, label %exit.1
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ br label %loop.header
+exit.1:
+ ret i16 0
+exit.2:
+ ret i16 1
+}
+
+define i32 @opt_for_size(ptr %p, i32 %n) optsize {
+; CHECK: LV: Checking a loop in 'opt_for_size'
+; CHECK: LV: No epilogue to apply tail-folding for.
+; CHECK-NEXT: LV: Fallback to a normal epilogue
+entry:
+ br label %for.body
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
+ %0 = load i32, ptr %arrayidx, align 1
+ %cmp1 = icmp eq i32 %0, 0
+ %sel = select i1 %cmp1, i32 2, i32 1
+ store i32 %sel, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, %n
+ br i1 %exitcond, label %for.end, label %for.body
+for.end:
+ ret i32 0
+}
>From 29573b8b27a13c5b90f0a551b29bf1347fc439ff Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 1 May 2026 13:13:13 +0000
Subject: [PATCH 11/11] resolve review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 26 ++++++++++---------
.../LoopVectorize/fold-epilogue-tail.ll | 1 -
2 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ffb46955a399a..65278fbcf6b81 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7588,31 +7588,32 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
/// \return CM_EpilogueNotNeededFoldEpilogueTail if epilogue tail-folding
/// is possible, otherwise CM_EpilogueAllowed.
static EpilogueLowering
-getEpilogueTailLowering(const LoopVectorizationCostModel &CM) {
+getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
+ OptimizationRemarkEmitter *ORE) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (TailFoldingPolicy != TailFoldingPolicyTy::PreferFoldEpilogueTail)
- // fallback to a normal epilogue.
return CM_EpilogueAllowed;
if (!EnableEpilogueVectorization) {
- LLVM_DEBUG(dbgs() << "LV: Options conflict, epilogue vectorization is "
- "disallowed while epilogue tail-folding allowed!\n");
- LLVM_DEBUG(dbgs() << "LV: Fallback to a normal epilogue\n");
+ reportVectorizationInfo(
+ "LV: Options conflict, epilogue vectorization is disallowed while "
+ "epilogue tail-folding allowed!\n",
+ "UnsupportedTailFoldingPolicy", ORE, L);
return CM_EpilogueAllowed;
}
// If scalar epilogue is explicitly required, we can't apply TF.
- if (CM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
+ if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
- "scalar epilogue is required\n");
- LLVM_DEBUG(dbgs() << "LV: Fallback to a normal epilogue\n");
+ "scalar epilogue is required\n"
+ "LV: Fallback to a normal epilogue\n");
return CM_EpilogueAllowed;
}
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
- if (!CM.isEpilogueAllowed()) {
- LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n");
- LLVM_DEBUG(dbgs() << "LV: Fallback to a normal epilogue\n");
+ if (!MainCM.isEpilogueAllowed()) {
+ LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n"
+ "LV: Fallback to a normal epilogue\n");
return CM_EpilogueAllowed;
}
@@ -8483,7 +8484,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
Hints, ORE);
- EpilogueLowering EpilogueTailLoweringStatus = getEpilogueTailLowering(CM);
+ EpilogueLowering EpilogueTailLoweringStatus =
+ getEpilogueTailLowering(CM, L, ORE);
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldEpilogueTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 0279ce99143f4..7f32f1bfd38ef 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -28,7 +28,6 @@ exit:
define void @epilogue_is_disabled(ptr %a, i64 %n) {
; CHECK-DISABLED-EPILOG: LV: Checking a loop in 'epilogue_is_disabled'
; CHECK-DISABLED-EPILOG: LV: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
-; CHECK-DISABLED-EPILOG-NEXT: LV: Fallback to a normal epilogue
entry:
%cmp1 = icmp sgt i64 %n, 0
br i1 %cmp1, label %for.body, label %for.end
More information about the llvm-commits
mailing list