[llvm] [LV] Add eligibility checks for epilogue tail-folding. (PR #217364)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 04:01:31 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/217364
>From 1d309d714f72e9914244f6a5973462133e8e0522 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 19 Aug 2026 15:11:05 +0100
Subject: [PATCH 1/4] [LV] Add tests for excluded cases in epiloge tail-folding
Cases that are excluded in epilogue tail-folding:
- When scalar epilogue is required.
- TC is too small.
- For now, some reduction cases that could require extra overhead.
- When epilogue vectorization is disabled.
- When VF/EpilogueVF are not forced.
- Outer loop.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 59 +++++-
.../LoopVectorize/fold-epilogue-tail.ll | 168 ++++++++++++++----
2 files changed, 181 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b45b8af5bf846..908e7ce5295e3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7139,7 +7139,9 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
/// otherwise CM_EpilogueAllowed.
static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
- OptimizationRemarkEmitter *ORE) {
+ OptimizationRemarkEmitter *ORE,
+ LoopVectorizationLegality &LVL,
+ LoopVectorizeHints &Hints) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -7148,23 +7150,64 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
if (!EnableEpilogueVectorization) {
reportVectorizationInfo(
"Options conflict, epilogue vectorization is disallowed while "
- "epilogue tail-folding allowed!\n",
+ "epilogue tail-folding allowed!",
"UnsupportedEpilogueTailFoldingPolicy", ORE, L);
return CM_EpilogueAllowed;
}
+ if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
+ reportVectorizationInfo("For now, Epilogue tail-folding can't be "
+ "applied without forced main/epilogue loop VF",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (!L->isInnermost()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported for outer loop",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
// If scalar epilogue is explicitly required, we can't apply TF.
if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
- "scalar epilogue is required\n"
- "LV: Fall back to a normal epilogue\n");
+ reportVectorizationInfo(
+ "Epilogue tail-folding can't be applied because scalar epilogue is "
+ "required. Fall back to a normal epilogue",
+ "InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
- LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n"
- "LV: Fall back to a normal epilogue\n");
+ reportVectorizationInfo("No epilogue to apply tail-folding for. Fall back "
+ "to a normal epilogue",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ // For now epilogue TF is not supported for some kinds of reductions that
+ // results in more overhead
+ bool HasReductions = !LVL.getReductionVars().empty();
+ bool HasSelectCmpReductions =
+ HasReductions &&
+ any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
+ const RecurrenceDescriptor &RdxDesc = Reduction.second;
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
+ RecurrenceDescriptor::isFindIVRecurrenceKind(RK);
+ });
+ if (HasSelectCmpReductions) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported yet for select-cmp Reductions",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (L->getExitingBlock() != L->getLoopLatch()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported yet for early-exit loops",
+ "InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
@@ -7994,7 +8037,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE);
+ getEpilogueTailLowering(CM, L, ORE, LVL, Hints);
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 590907a5aff79..4d44ee20c89ee 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -1,23 +1,45 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 \
-; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s
-define void @test_epilogue_tf(ptr %A, i64 %n) {
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
+; RUN: --disable-output -force-vector-width=16 -epilogue-vectorization-force-VF=8 -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
+
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF=8 \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
+
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -enable-vplan-native-path --disable-output \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize \
+; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+
+
+define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is not supported yet
; CHECK: remark: <unknown>:0:0: The epilogue-tail-folding policy prefer-fold-tail is not supported yet, fall back to a normal epilogue
;
+; CHECK-DISABLED-EPILOG-LABEL: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
+;
+; CHECK-NO-FORCED-MAIN-VF-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-NO-FORCED-MAIN-VF: remark: <unknown>:0:0: For now, Epilogue tail-folding can't be applied without forced main/epilogue loop VF
+
+; CHECK-NO-FORCED-EPILOGUE-VF-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, Epilogue tail-folding can't be applied without forced main/epilogue loop VF
+;
+
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
- store i8 1, ptr %arrayidx, align 1
+ store i8 %val, ptr %arrayidx, align 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp ne i64 %iv.next, %n
br i1 %exitcond, label %for.body, label %exit
@@ -26,29 +48,9 @@ exit:
ret void
}
-define void @epilogue_is_disabled(ptr %a, i64 %n) {
-; CHECK-DISABLED-EPILOG-LABEL: LV: Checking a loop in 'epilogue_is_disabled'
-; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
-;
-entry:
- br label %for.body
-
-for.body:
- %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
- store i32 1, ptr %arrayidx, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp ne i64 %indvars.iv.next, %n
- br i1 %exitcond, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
define i16 @require_scalar_epilogue(ptr %dst, i64 %x) {
-; CHECK-LABEL: LV: Checking a loop in 'require_scalar_epilogue'
-; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
-; CHECK-NEXT: LV: Fall back to a normal epilogue
+; CHECK-LABEL: Checking a loop in 'require_scalar_epilogue'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding can't be applied because scalar epilogue is required. Fall back to a normal epilogue
;
entry:
br label %loop.header
@@ -74,21 +76,17 @@ exit.2:
ret i16 1
}
-define i32 @opt_for_size(ptr %p, i32 %n) optsize {
-; CHECK-LABEL: LV: Checking a loop in 'opt_for_size'
-; CHECK: LV: No epilogue to apply tail-folding for.
-; CHECK-NEXT: LV: Fall back to a normal epilogue
+define i32 @opt_for_size(ptr %p, i32 %n, i8 %val) optsize {
+; CHECK-LABEL: Checking a loop in 'opt_for_size'
+; CHECK: remark: <unknown>:0:0: No epilogue to apply tail-folding for. Fall back to a normal epilogue
;
entry:
br label %for.body
for.body:
%iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
- %0 = load i32, ptr %arrayidx, align 1
- %cmp1 = icmp eq i32 %0, 0
- %sel = select i1 %cmp1, i32 2, i32 1
- store i32 %sel, ptr %arrayidx, align 1
+ %arrayidx = getelementptr inbounds i8, ptr %p, i32 %iv
+ store i8 %val, ptr %arrayidx, align 1
%inc = add nsw i32 %iv, 1
%exitcond = icmp eq i32 %inc, %n
br i1 %exitcond, label %for.end, label %for.body
@@ -96,3 +94,97 @@ for.body:
for.end:
ret i32 0
}
+
+define i32 @low_tc(ptr %p, i8 %val) {
+; CHECK-LABEL: Checking a loop in 'low_tc'
+; CHECK: remark: <unknown>:0:0: No epilogue to apply tail-folding for. Fall back to a normal epilogue
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %p, i32 %iv
+ store i8 %val, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, 8
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret i32 0
+}
+
+define i32 @any-of-redc(ptr %src, i64 %n) {
+; CHECK-LABEL: Checking a loop in 'any-of-redc'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for select-cmp Reductions
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %select, %loop ]
+ %gep = getelementptr inbounds i8, ptr %src, i64 %iv
+ %load = load i8, ptr %gep, align 1
+ %icmp = icmp eq i8 %load, 0
+ %select = select i1 %icmp, i32 1, i32 %red
+ %iv.next = add i64 %iv, 1
+ %icmp3 = icmp eq i64 %iv, %n
+ br i1 %icmp3, label %exit, label %loop
+
+exit:
+ ret i32 %select
+}
+
+define i1 @early_exit(ptr %A, i64 %n, i8 %find) {
+; CHECK-LABEL: LV: Checking a loop in 'early_exit'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ %val = load i8, ptr %arrayidx, align 1
+ %exitcond = icmp eq i8 %val, %find
+ br i1 %exitcond, label %exit, label %cont
+
+cont:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %contcond = icmp ne i64 %iv.next, %n
+ br i1 %contcond, label %for.body, label %exit
+exit:
+ ret i1 %exitcond
+}
+
+define void @test_outer_loop(ptr %A, i64 %m) {
+; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
+; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+ %gep = getelementptr inbounds i8, ptr %A, i64 %iv.inner
+ store i8 0, ptr %gep, align 1
+ %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+ %inner.ec = icmp eq i64 %iv.inner.next, 8
+ br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+ %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+ %outer.ec = icmp eq i64 %iv.outer.next, %m
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+ ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.vectorize.enable"}
+
>From aa95ba329e621c43379657f81e6273d322dad7d5 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 24 Aug 2026 20:07:00 +0100
Subject: [PATCH 2/4] resolve review comments and exclude case when mainVF is
<= epilogueVF
---
.../Transforms/Vectorize/LoopVectorize.cpp | 33 +++-----
.../LoopVectorize/fold-epilogue-tail.ll | 77 +++++++++++--------
2 files changed, 59 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 908e7ce5295e3..5f9b3cdf24894 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7156,12 +7156,19 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
}
if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
- reportVectorizationInfo("For now, Epilogue tail-folding can't be "
+ reportVectorizationInfo("For now, epilogue tail-folding can't be "
"applied without forced main/epilogue loop VF",
"UnsupportedEpilogueTailFoldingPolicy", ORE, L);
return CM_EpilogueAllowed;
}
+ if (ElementCount::isKnownLE(Hints.getWidth(), EpilogueVectorizationForceVF)) {
+ reportVectorizationInfo("For now, epilogue tail-folding can't be "
+ "applied when mainVF <= epilogueVF",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
if (!L->isInnermost()) {
reportVectorizationInfo(
"Epilogue tail-folding is not supported for outer loop",
@@ -7180,31 +7187,15 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
- reportVectorizationInfo("No epilogue to apply tail-folding for. Fall back "
- "to a normal epilogue",
- "InvalidTailFoldedEpilogue", ORE, L);
- return CM_EpilogueAllowed;
- }
-
- // For now epilogue TF is not supported for some kinds of reductions that
- // results in more overhead
- bool HasReductions = !LVL.getReductionVars().empty();
- bool HasSelectCmpReductions =
- HasReductions &&
- any_of(LVL.getReductionVars(), [](auto &Reduction) -> bool {
- const RecurrenceDescriptor &RdxDesc = Reduction.second;
- RecurKind RK = RdxDesc.getRecurrenceKind();
- return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
- RecurrenceDescriptor::isFindIVRecurrenceKind(RK);
- });
- if (HasSelectCmpReductions) {
reportVectorizationInfo(
- "Epilogue tail-folding is not supported yet for select-cmp Reductions",
+ "Not applying tail-folding to the epilogue, since tail-folding is "
+ "already requested for the main vector loop.",
"InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
- if (L->getExitingBlock() != L->getLoopLatch()) {
+ if (L->getExitingBlock() != L->getLoopLatch() ||
+ LVL.hasUncountableEarlyExit()) {
reportVectorizationInfo(
"Epilogue tail-folding is not supported yet for early-exit loops",
"InvalidTailFoldedEpilogue", ORE, L);
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 4d44ee20c89ee..194f075abc176 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -7,11 +7,18 @@
; RUN: --disable-output -force-vector-width=16 -epilogue-vectorization-force-VF=8 -epilogue-tail-folding-policy=prefer-fold-tail \
; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF=8 \
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
+
; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 \
-; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
+; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF=8 \
-; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=8 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-VFs
+
+; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
+; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=16 -epilogue-vectorization-force-VF=8 \
+; RUN: -enable-early-exit-vectorization-with-side-effects < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EARLY-EXIT
; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -enable-vplan-native-path --disable-output \
; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize \
@@ -27,10 +34,13 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
;
; CHECK-NO-FORCED-MAIN-VF-LABEL: Checking a loop in 'test_epilogue_tf'
-; CHECK-NO-FORCED-MAIN-VF: remark: <unknown>:0:0: For now, Epilogue tail-folding can't be applied without forced main/epilogue loop VF
+; CHECK-NO-FORCED-MAIN-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
; CHECK-NO-FORCED-EPILOGUE-VF-LABEL: Checking a loop in 'test_epilogue_tf'
-; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, Epilogue tail-folding can't be applied without forced main/epilogue loop VF
+; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
+;
+; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when mainVF <= epilogueVF
;
entry:
@@ -78,7 +88,7 @@ exit.2:
define i32 @opt_for_size(ptr %p, i32 %n, i8 %val) optsize {
; CHECK-LABEL: Checking a loop in 'opt_for_size'
-; CHECK: remark: <unknown>:0:0: No epilogue to apply tail-folding for. Fall back to a normal epilogue
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since tail-folding is already requested for the main vector loop.
;
entry:
br label %for.body
@@ -97,7 +107,7 @@ for.end:
define i32 @low_tc(ptr %p, i8 %val) {
; CHECK-LABEL: Checking a loop in 'low_tc'
-; CHECK: remark: <unknown>:0:0: No epilogue to apply tail-folding for. Fall back to a normal epilogue
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since tail-folding is already requested for the main vector loop.
;
entry:
br label %for.body
@@ -114,30 +124,9 @@ for.end:
ret i32 0
}
-define i32 @any-of-redc(ptr %src, i64 %n) {
-; CHECK-LABEL: Checking a loop in 'any-of-redc'
-; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for select-cmp Reductions
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %red = phi i32 [ 0, %entry ], [ %select, %loop ]
- %gep = getelementptr inbounds i8, ptr %src, i64 %iv
- %load = load i8, ptr %gep, align 1
- %icmp = icmp eq i8 %load, 0
- %select = select i1 %icmp, i32 1, i32 %red
- %iv.next = add i64 %iv, 1
- %icmp3 = icmp eq i64 %iv, %n
- br i1 %icmp3, label %exit, label %loop
-
-exit:
- ret i32 %select
-}
-
define i1 @early_exit(ptr %A, i64 %n, i8 %find) {
-; CHECK-LABEL: LV: Checking a loop in 'early_exit'
-; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
+; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'early_exit'
+; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
;
entry:
br label %for.body
@@ -157,6 +146,34 @@ exit:
ret i1 %exitcond
}
+; For this function, the check line is not related to epilogue tail-folding, but when vectorizing this case gets supported,
+; the check line should be changed to: Epilogue tail-folding is not supported yet for early-exit loops, same as the case above.
+define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %src, ptr align 4 dereferenceable(80) noalias %dst, ptr align 4 dereferenceable(80) readonly %pred) {
+; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'combined_exit_conditions'
+; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: loop not vectorized: Cannot vectorize uncountable loop
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %src.ptr = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
+ %data = load i32, ptr %src.ptr, align 4
+ %add = add nsw i32 %data, 1
+ %dst.ptr = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %iv
+ store i32 %add, ptr %dst.ptr, align 4
+ %ee.ptr = getelementptr inbounds nuw [4 x i8], ptr %pred, i64 %iv
+ %ee.val = load i32, ptr %ee.ptr, align 4
+ %ee.cmp = icmp ne i32 %ee.val, 0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cmp = icmp eq i64 %iv.next, 20
+ %combined.cond = select i1 %ee.cmp, i1 true, i1 %counted.cmp
+ br i1 %combined.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
define void @test_outer_loop(ptr %A, i64 %m) {
; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
>From 77f1ccd3f0a9f0eef724725c76fe404bb4ef62b7 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Wed, 26 Aug 2026 17:42:56 +0100
Subject: [PATCH 3/4] improve log msg and shorten the RUN commands in lit test
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 ++--
.../LoopVectorize/fold-epilogue-tail.ll | 34 ++++++++-----------
2 files changed, 18 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5f9b3cdf24894..5bd4c795da97e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7187,10 +7187,9 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
- reportVectorizationInfo(
- "Not applying tail-folding to the epilogue, since tail-folding is "
- "already requested for the main vector loop.",
- "InvalidTailFoldedEpilogue", ORE, L);
+ reportVectorizationInfo("Not applying tail-folding to the epilogue, since "
+ "no epilogue allowed.",
+ "InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 194f075abc176..cd099ab88fbc3 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -1,28 +1,24 @@
; REQUIRES: asserts
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s
+; DEFINE: %{cmd} = opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; DEFINE: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -force-vector-width=16 -epilogue-vectorization-force-VF=8 -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF=8 \
-; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-epilogue-vectorization=false \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -force-vector-width=16 \
-; RUN: -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
+; RUN: %{cmd} -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=8 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-VFs
+; RUN: %{cmd} -force-vector-width=16 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output -epilogue-tail-folding-policy=prefer-fold-tail \
-; RUN: -pass-remarks-analysis=loop-vectorize -force-vector-width=16 -epilogue-vectorization-force-VF=8 \
-; RUN: -enable-early-exit-vectorization-with-side-effects < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EARLY-EXIT
+; RUN: %{cmd} -force-vector-width=8 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-VFs
-; RUN: opt -S -p loop-vectorize -debug-only=loop-vectorize -enable-vplan-native-path --disable-output \
-; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize \
-; RUN: -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-early-exit-vectorization-with-side-effects \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EARLY-EXIT
+
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
@@ -88,7 +84,7 @@ exit.2:
define i32 @opt_for_size(ptr %p, i32 %n, i8 %val) optsize {
; CHECK-LABEL: Checking a loop in 'opt_for_size'
-; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since tail-folding is already requested for the main vector loop.
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue allowed.
;
entry:
br label %for.body
@@ -107,7 +103,7 @@ for.end:
define i32 @low_tc(ptr %p, i8 %val) {
; CHECK-LABEL: Checking a loop in 'low_tc'
-; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since tail-folding is already requested for the main vector loop.
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue allowed.
;
entry:
br label %for.body
>From ae4c55493d2d08e8ea12e6d7096e5b61c00a0c1b Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Thu, 27 Aug 2026 11:48:04 +0100
Subject: [PATCH 4/4] improve logs
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 +++---
llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5bd4c795da97e..1833aa1ee9726 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7163,8 +7163,8 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
}
if (ElementCount::isKnownLE(Hints.getWidth(), EpilogueVectorizationForceVF)) {
- reportVectorizationInfo("For now, epilogue tail-folding can't be "
- "applied when mainVF <= epilogueVF",
+ reportVectorizationInfo("For now, epilogue tail-folding can't be applied "
+ "when VF of the main loop <= VF of the epilogue",
"UnsupportedEpilogueTailFoldingPolicy", ORE, L);
return CM_EpilogueAllowed;
}
@@ -7188,7 +7188,7 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
reportVectorizationInfo("Not applying tail-folding to the epilogue, since "
- "no epilogue allowed.",
+ "no epilogue is allowed.",
"InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index cd099ab88fbc3..78aa3eda9d952 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -36,7 +36,7 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
;
; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
-; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when mainVF <= epilogueVF
+; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
;
entry:
@@ -84,7 +84,7 @@ exit.2:
define i32 @opt_for_size(ptr %p, i32 %n, i8 %val) optsize {
; CHECK-LABEL: Checking a loop in 'opt_for_size'
-; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue allowed.
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue is allowed
;
entry:
br label %for.body
@@ -103,7 +103,7 @@ for.end:
define i32 @low_tc(ptr %p, i8 %val) {
; CHECK-LABEL: Checking a loop in 'low_tc'
-; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue allowed.
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue is allowed.
;
entry:
br label %for.body
More information about the llvm-commits
mailing list