[llvm] 0fb7d02 - [LV] Add eligibility checks for epilogue tail-folding. (#217364)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 04:45:41 PDT 2026
Author: Hassnaa Hamdi
Date: 2026-08-27T12:45:36+01:00
New Revision: 0fb7d02901dbf09be9c4e82d76271bf520a4ad19
URL: https://github.com/llvm/llvm-project/commit/0fb7d02901dbf09be9c4e82d76271bf520a4ad19
DIFF: https://github.com/llvm/llvm-project/commit/0fb7d02901dbf09be9c4e82d76271bf520a4ad19.diff
LOG: [LV] Add eligibility checks for epilogue tail-folding. (#217364)
For now, epilogue tail-folding is restricted to innermost loops with
explicitly forced main and epilogue VFs, where mainVF > epilogueVF.
Reject cases requiring a scalar epilogue, cases where the main vector
loop is already tail-folded, and early-exit loops.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a5dff6d0267b0..1f153afd6bedd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7139,7 +7139,9 @@ getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
/// otherwise CM_EpilogueAllowed.
static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
- OptimizationRemarkEmitter *ORE) {
+ OptimizationRemarkEmitter *ORE,
+ LoopVectorizationLegality &LVL,
+ LoopVectorizeHints &Hints) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -7148,23 +7150,54 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
if (!EnableEpilogueVectorization) {
reportVectorizationInfo(
"Options conflict, epilogue vectorization is disallowed while "
- "epilogue tail-folding allowed!\n",
+ "epilogue tail-folding allowed!",
"UnsupportedEpilogueTailFoldingPolicy", ORE, L);
return CM_EpilogueAllowed;
}
+ if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
+ reportVectorizationInfo("For now, epilogue tail-folding can't be "
+ "applied without forced main/epilogue loop VF",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (ElementCount::isKnownLE(Hints.getWidth(), EpilogueVectorizationForceVF)) {
+ reportVectorizationInfo("For now, epilogue tail-folding can't be applied "
+ "when VF of the main loop <= VF of the epilogue",
+ "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (!L->isInnermost()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported for outer loop",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
// If scalar epilogue is explicitly required, we can't apply TF.
if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
- LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
- "scalar epilogue is required\n"
- "LV: Fall back to a normal epilogue\n");
+ reportVectorizationInfo(
+ "Epilogue tail-folding can't be applied because scalar epilogue is "
+ "required. Fall back to a normal epilogue",
+ "InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
// If having epilogue is NOT allowed, then no epilogue to apply TF for.
if (!MainCM.isEpilogueAllowed()) {
- LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n"
- "LV: Fall back to a normal epilogue\n");
+ reportVectorizationInfo("Not applying tail-folding to the epilogue, since "
+ "no epilogue is allowed.",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (L->getExitingBlock() != L->getLoopLatch() ||
+ LVL.hasUncountableEarlyExit()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported yet for early-exit loops",
+ "InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
@@ -7994,7 +8027,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
ORE);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(CM, L, ORE);
+ getEpilogueTailLowering(CM, L, ORE, LVL, Hints);
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 590907a5aff79..78aa3eda9d952 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -1,23 +1,51 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 | FileCheck %s
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-epilogue-vectorization=false \
-; RUN: --disable-output -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize 2>&1 \
-; RUN: | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+; DEFINE: %{cmd} = opt -S -p loop-vectorize -debug-only=loop-vectorize --disable-output \
+; DEFINE: -epilogue-tail-folding-policy=prefer-fold-tail -pass-remarks-analysis=loop-vectorize
-define void @test_epilogue_tf(ptr %A, i64 %n) {
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s
+
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-epilogue-vectorization=false \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EPILOG
+
+; RUN: %{cmd} -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-MAIN-VF
+
+; RUN: %{cmd} -force-vector-width=16 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-FORCED-EPILOGUE-VF
+
+; RUN: %{cmd} -force-vector-width=8 -epilogue-vectorization-force-VF=8 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-VFs
+
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-early-exit-vectorization-with-side-effects \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-DISABLED-EARLY-EXIT
+
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+
+
+define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is not supported yet
; CHECK: remark: <unknown>:0:0: The epilogue-tail-folding policy prefer-fold-tail is not supported yet, fall back to a normal epilogue
;
+; CHECK-DISABLED-EPILOG-LABEL: LV: Checking a loop in 'test_epilogue_tf'
+; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
+;
+; CHECK-NO-FORCED-MAIN-VF-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-NO-FORCED-MAIN-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
+
+; CHECK-NO-FORCED-EPILOGUE-VF-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-NO-FORCED-EPILOGUE-VF: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied without forced main/epilogue loop VF
+;
+; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
+;
+
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
- store i8 1, ptr %arrayidx, align 1
+ store i8 %val, ptr %arrayidx, align 1
%iv.next = add nuw nsw i64 %iv, 1
%exitcond = icmp ne i64 %iv.next, %n
br i1 %exitcond, label %for.body, label %exit
@@ -26,29 +54,9 @@ exit:
ret void
}
-define void @epilogue_is_disabled(ptr %a, i64 %n) {
-; CHECK-DISABLED-EPILOG-LABEL: LV: Checking a loop in 'epilogue_is_disabled'
-; CHECK-DISABLED-EPILOG: remark: <unknown>:0:0: Options conflict, epilogue vectorization is disallowed while epilogue tail-folding allowed!
-;
-entry:
- br label %for.body
-
-for.body:
- %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
- store i32 1, ptr %arrayidx, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp ne i64 %indvars.iv.next, %n
- br i1 %exitcond, label %for.body, label %for.end
-
-for.end:
- ret void
-}
-
define i16 @require_scalar_epilogue(ptr %dst, i64 %x) {
-; CHECK-LABEL: LV: Checking a loop in 'require_scalar_epilogue'
-; CHECK: LV: Epilogue tail-folding can't be applied because scalar epilogue is required
-; CHECK-NEXT: LV: Fall back to a normal epilogue
+; CHECK-LABEL: Checking a loop in 'require_scalar_epilogue'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding can't be applied because scalar epilogue is required. Fall back to a normal epilogue
;
entry:
br label %loop.header
@@ -74,21 +82,17 @@ exit.2:
ret i16 1
}
-define i32 @opt_for_size(ptr %p, i32 %n) optsize {
-; CHECK-LABEL: LV: Checking a loop in 'opt_for_size'
-; CHECK: LV: No epilogue to apply tail-folding for.
-; CHECK-NEXT: LV: Fall back to a normal epilogue
+define i32 @opt_for_size(ptr %p, i32 %n, i8 %val) optsize {
+; CHECK-LABEL: Checking a loop in 'opt_for_size'
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue is allowed
;
entry:
br label %for.body
for.body:
%iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
- %arrayidx = getelementptr inbounds i32, ptr %p, i32 %iv
- %0 = load i32, ptr %arrayidx, align 1
- %cmp1 = icmp eq i32 %0, 0
- %sel = select i1 %cmp1, i32 2, i32 1
- store i32 %sel, ptr %arrayidx, align 1
+ %arrayidx = getelementptr inbounds i8, ptr %p, i32 %iv
+ store i8 %val, ptr %arrayidx, align 1
%inc = add nsw i32 %iv, 1
%exitcond = icmp eq i32 %inc, %n
br i1 %exitcond, label %for.end, label %for.body
@@ -96,3 +100,104 @@ for.body:
for.end:
ret i32 0
}
+
+define i32 @low_tc(ptr %p, i8 %val) {
+; CHECK-LABEL: Checking a loop in 'low_tc'
+; CHECK: remark: <unknown>:0:0: Not applying tail-folding to the epilogue, since no epilogue is allowed.
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %arrayidx = getelementptr inbounds i8, ptr %p, i32 %iv
+ store i8 %val, ptr %arrayidx, align 1
+ %inc = add nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %inc, 8
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret i32 0
+}
+
+define i1 @early_exit(ptr %A, i64 %n, i8 %find) {
+; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'early_exit'
+; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv
+ %val = load i8, ptr %arrayidx, align 1
+ %exitcond = icmp eq i8 %val, %find
+ br i1 %exitcond, label %exit, label %cont
+
+cont:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %contcond = icmp ne i64 %iv.next, %n
+ br i1 %contcond, label %for.body, label %exit
+exit:
+ ret i1 %exitcond
+}
+
+; For this function, the check line is not related to epilogue tail-folding, but when vectorizing this case gets supported,
+; the check line should be changed to: Epilogue tail-folding is not supported yet for early-exit loops, same as the case above.
+define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %src, ptr align 4 dereferenceable(80) noalias %dst, ptr align 4 dereferenceable(80) readonly %pred) {
+; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'combined_exit_conditions'
+; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: loop not vectorized: Cannot vectorize uncountable loop
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %src.ptr = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
+ %data = load i32, ptr %src.ptr, align 4
+ %add = add nsw i32 %data, 1
+ %dst.ptr = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %iv
+ store i32 %add, ptr %dst.ptr, align 4
+ %ee.ptr = getelementptr inbounds nuw [4 x i8], ptr %pred, i64 %iv
+ %ee.val = load i32, ptr %ee.ptr, align 4
+ %ee.cmp = icmp ne i32 %ee.val, 0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cmp = icmp eq i64 %iv.next, 20
+ %combined.cond = select i1 %ee.cmp, i1 true, i1 %counted.cmp
+ br i1 %combined.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @test_outer_loop(ptr %A, i64 %m) {
+; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
+; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+ br label %inner
+
+inner:
+ %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+ %gep = getelementptr inbounds i8, ptr %A, i64 %iv.inner
+ store i8 0, ptr %gep, align 1
+ %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+ %inner.ec = icmp eq i64 %iv.inner.next, 8
+ br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+ %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+ %outer.ec = icmp eq i64 %iv.outer.next, %m
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+ ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.vectorize.enable"}
+
More information about the llvm-commits
mailing list