[llvm] [LV][EpilogueTF] Exclude loops with recurrences or alias-masking (PR #224835)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 09:48:14 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/224835
>From c36808ccfea0ca7d0670fcef79e36ba46d3cc3a9 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 18 Sep 2026 22:11:16 +0100
Subject: [PATCH 1/2] [LV] exclude reduction and fixed-order recurrences cases
from tail-folded vector epilogue
---
.../Transforms/Vectorize/LoopVectorize.cpp | 21 +++++++-
.../LoopVectorize/fold-epilogue-tail.ll | 49 ++++++++++++++++++-
2 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 716c73a7c63b3..dee0edf8d1ca7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3365,7 +3365,7 @@ static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
OptimizationRemarkEmitter *ORE,
LoopVectorizationLegality &LVL,
- LoopVectorizeHints &Hints) {
+ const LoopVectorizeHints &Hints) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -3425,6 +3425,25 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ if (ForcePartialAliasingVectorization) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with alias masking",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (!LVL.getReductionVars().empty()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with reductions",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+ if (!LVL.getFixedOrderRecurrences().empty()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with fixed-order recurrence",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
// We can apply tail-folding on the vectorized epilogue loop.
return CM_EpilogueNotNeededFoldTail;
}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 78aa3eda9d952..0fff5b9e889ad 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -20,6 +20,8 @@
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -force-partial-aliasing-vectorization \
+; RUN: -force-target-supports-masked-memory-ops < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ALIAS-MASK
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
@@ -38,7 +40,9 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
;
-
+; CHECK-ALIAS-MASK-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-ALIAS-MASK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with alias masking
+;
entry:
br label %for.body
@@ -120,6 +124,49 @@ for.end:
ret i32 0
}
+define i64 @find_last_offset_wide_canonical_iv(ptr %A, i64 %n) {
+; CHECK-LABEL: Checking a loop in 'find_last_offset_wide_canonical_iv'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with reductions
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %c = icmp eq i32 %l, 11
+ %sel = select i1 %c, i64 %iv, i64 %red
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i64 %sel
+}
+
+define i32 @fixed-order-recurrence(ptr %src) {
+; CHECK-LABEL: Checking a loop in 'fixed-order-recurrence'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with fixed-order recurrence
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+ %previous = phi i32 [ 0, %entry ], [ %ld, %for.body ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %i
+ %ld = load i32, ptr %gep, align 4
+ %inc = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %inc, 23
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %result = phi i32 [ %previous, %for.body ]
+ ret i32 %result
+}
+
define i1 @early_exit(ptr %A, i64 %n, i8 %find) {
; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'early_exit'
; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
>From b5951fec38c7c0723dd878513a76c1fbb1bcf090 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 21 Sep 2026 17:47:54 +0100
Subject: [PATCH 2/2] add blank lines
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dee0edf8d1ca7..6a225e89e337e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3438,12 +3438,14 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
"InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
+
if (!LVL.getFixedOrderRecurrences().empty()) {
reportVectorizationInfo(
"Epilogue tail-folding is not supported with fixed-order recurrence",
"InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
+
// We can apply tail-folding on the vectorized epilogue loop.
return CM_EpilogueNotNeededFoldTail;
}
More information about the llvm-commits
mailing list