[llvm] [LV] Don't tail-fold the epilogue with EVL-based tail-folding (PR #228452)

[Hassnaa Hamdi via llvm-commits]

https://github.com/hassnaaHamdi created https://github.com/llvm/llvm-project/pull/228452

Epilogue tail-folding isn't supported yet with the `DataWithEVL`
tail-folding style. The epilogue plan is selected by duplicating an
existing VPlan, and some recipes used only by EVL-based tail-folding
don't implement clone() yet, so cloning a plan that contains them hits
llvm_unreachable.  This affects targets that prefer `DataWithEVL`, such as
RISC-V.

Until those recipes implement clone(), fall back to a normal epilogue
when the preferred (or forced) tail-folding style is DataWithEVL. This
bail-out is based on the preferred style rather than the style that
would actually be chosen, so it also rejects fixed-width epilogue VFs,
where EVL isn't used. It will be refined once the epilogue's chosen
tail-folding style is known when epilogue TF gets supported.

>From 38061e56dc38fc4448c1cbca273d03179c9aac02 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 2 Oct 2026 14:03:45 +0000
Subject: [PATCH] [LV] Don't tail-fold the epilogue with EVL-based tail-folding

---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 14 ++++++++
 .../LoopVectorize/RISCV/fold-epilogue-tail.ll | 35 +++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/fold-epilogue-tail.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 45edf702d70920..b264a9950d31c8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3490,6 +3490,20 @@ static EpilogueLowering getEpilogueTailLowering(
     return CM_EpilogueAllowed;
   }
 
+  // TODO: This is conservative: it rejects any target that prefers EVL, even
+  // for fixed-width epilogue VFs where EVL won't be chosen. Move this check to
+  // where the epilogue's TF style is known once epilogue TF is supported.
+  TailFoldingStyle TFStyle = TTI->getPreferredTailFoldingStyle();
+  if (ForceTailFoldingStyle.getNumOccurrences())
+    TFStyle = ForceTailFoldingStyle.getValue();
+  // TODO: Remove once EVL recipes support cloning.
+  if (TFStyle == TailFoldingStyle::DataWithEVL) {
+    reportVectorizationInfo("Epilogue tail-folding is not supported yet with "
+                            "EVL-based tail-folding",
+                            "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
+    return CM_EpilogueAllowed;
+  }
+
   // We can apply tail-folding on the vectorized epilogue loop.
   return CM_EpilogueNotNeededFoldTail;
 }
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/RISCV/fold-epilogue-tail.ll
new file mode 100644
index 00000000000000..41218a749f7d1b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/fold-epilogue-tail.ll
@@ -0,0 +1,35 @@
+; REQUIRES: asserts
+
+; RUN: opt -passes=loop-vectorize -force-vector-width=16 -tail-folding-policy=dont-fold-tail \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF=8 \
+; RUN: -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize \
+; RUN: -disable-output -mtriple=riscv64 -mattr=+v < %s 2>&1 | FileCheck %s --check-prefix=CHECK-FIXED-VF
+
+; RUN: opt -passes=loop-vectorize -force-vector-width="vscale x 4" -tail-folding-policy=dont-fold-tail \
+; RUN: -epilogue-tail-folding-policy=prefer-fold-tail -epilogue-vectorization-force-VF="vscale x 2" \
+; RUN: -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize \
+; RUN: -disable-output -mtriple=riscv64 -mattr=+v < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE_VF
+
+define void @test_epilogue_tf(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-FIXED-VF-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-FIXED-VF: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet with EVL-based tail-folding
+
+; CHECK-SCALABLE_VF-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-SCALABLE_VF: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet with EVL-based tail-folding
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.a = getelementptr i32, ptr %a, i64 %iv
+  %l = load i32, ptr %gep.a, align 4
+  %gep.b = getelementptr i32, ptr %b, i64 %iv
+  store i32 %l, ptr %gep.b, align 4
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}