[llvm] [LV][EpilogueTF] Exclude loops with recurrences or alias-masking (PR #224835)
Hassnaa Hamdi via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 05:44:34 PDT 2026
https://github.com/hassnaaHamdi updated https://github.com/llvm/llvm-project/pull/224835
>From c36808ccfea0ca7d0670fcef79e36ba46d3cc3a9 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Fri, 18 Sep 2026 22:11:16 +0100
Subject: [PATCH 1/3] [LV] exclude reduction and fixed-order recurrences cases
from tail-folded vector epilogue
---
.../Transforms/Vectorize/LoopVectorize.cpp | 21 +++++++-
.../LoopVectorize/fold-epilogue-tail.ll | 49 ++++++++++++++++++-
2 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 716c73a7c63b3..dee0edf8d1ca7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3365,7 +3365,7 @@ static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
OptimizationRemarkEmitter *ORE,
LoopVectorizationLegality &LVL,
- LoopVectorizeHints &Hints) {
+ const LoopVectorizeHints &Hints) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -3425,6 +3425,25 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ if (ForcePartialAliasingVectorization) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with alias masking",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
+ if (!LVL.getReductionVars().empty()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with reductions",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+ if (!LVL.getFixedOrderRecurrences().empty()) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with fixed-order recurrence",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
// We can apply tail-folding on the vectorized epilogue loop.
return CM_EpilogueNotNeededFoldTail;
}
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 78aa3eda9d952..0fff5b9e889ad 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -20,6 +20,8 @@
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -enable-vplan-native-path \
; RUN: < %s 2>&1 | FileCheck %s --check-prefix=CHECK-OUTER-LOOP
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -force-partial-aliasing-vectorization \
+; RUN: -force-target-supports-masked-memory-ops < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ALIAS-MASK
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
@@ -38,7 +40,9 @@ define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-INVALID-VFs-LABEL: Checking a loop in 'test_epilogue_tf'
; CHECK-INVALID-VFs: remark: <unknown>:0:0: For now, epilogue tail-folding can't be applied when VF of the main loop <= VF of the epilogue
;
-
+; CHECK-ALIAS-MASK-LABEL: Checking a loop in 'test_epilogue_tf'
+; CHECK-ALIAS-MASK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with alias masking
+;
entry:
br label %for.body
@@ -120,6 +124,49 @@ for.end:
ret i32 0
}
+define i64 @find_last_offset_wide_canonical_iv(ptr %A, i64 %n) {
+; CHECK-LABEL: Checking a loop in 'find_last_offset_wide_canonical_iv'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with reductions
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i32, ptr %A, i64 %iv
+ %l = load i32, ptr %gep, align 4
+ %c = icmp eq i32 %l, 11
+ %sel = select i1 %c, i64 %iv, i64 %red
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i64 %sel
+}
+
+define i32 @fixed-order-recurrence(ptr %src) {
+; CHECK-LABEL: Checking a loop in 'fixed-order-recurrence'
+; CHECK: remark: <unknown>:0:0: Epilogue tail-folding is not supported with fixed-order recurrence
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+ %previous = phi i32 [ 0, %entry ], [ %ld, %for.body ]
+ %gep = getelementptr inbounds i32, ptr %src, i64 %i
+ %ld = load i32, ptr %gep, align 4
+ %inc = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %inc, 23
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %result = phi i32 [ %previous, %for.body ]
+ ret i32 %result
+}
+
define i1 @early_exit(ptr %A, i64 %n, i8 %find) {
; CHECK-DISABLED-EARLY-EXIT-LABEL: LV: Checking a loop in 'early_exit'
; CHECK-DISABLED-EARLY-EXIT: remark: <unknown>:0:0: Epilogue tail-folding is not supported yet for early-exit loops
>From b5951fec38c7c0723dd878513a76c1fbb1bcf090 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 21 Sep 2026 17:47:54 +0100
Subject: [PATCH 2/3] add blank lines
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dee0edf8d1ca7..6a225e89e337e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3438,12 +3438,14 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
"InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
+
if (!LVL.getFixedOrderRecurrences().empty()) {
reportVectorizationInfo(
"Epilogue tail-folding is not supported with fixed-order recurrence",
"InvalidTailFoldedEpilogue", ORE, L);
return CM_EpilogueAllowed;
}
+
// We can apply tail-folding on the vectorized epilogue loop.
return CM_EpilogueNotNeededFoldTail;
}
>From 9a5c0efcee96e43282e2c60bd1ce21c457d79901 Mon Sep 17 00:00:00 2001
From: Hassnaa Hamdi <hassnaa.hamdi at arm.com>
Date: Mon, 21 Sep 2026 23:58:16 +0100
Subject: [PATCH 3/3] exclude loops with interleaved accesses when target
doesn't support masked accesses
---
.../Transforms/Vectorize/LoopVectorize.cpp | 15 +++++++--
.../LoopVectorize/fold-epilogue-tail.ll | 33 +++++++++++++++++++
2 files changed, 46 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6a225e89e337e..1fefbc01665f5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3365,7 +3365,8 @@ static EpilogueLowering
getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
OptimizationRemarkEmitter *ORE,
LoopVectorizationLegality &LVL,
- const LoopVectorizeHints &Hints) {
+ const LoopVectorizeHints &Hints,
+ TargetTransformInfo *TTI) {
// Epilogue TF is only enabled when explicitly requested via command line.
if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
@@ -3425,6 +3426,16 @@ getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
return CM_EpilogueAllowed;
}
+ // The epilogue reuses the main loop's interleave groups, so it can't be
+ // tail-folded if the target can't mask interleaved accesses.
+ if (MainCM.InterleaveInfo.hasGroups() && !useMaskedInterleavedAccesses(*TTI)) {
+ reportVectorizationInfo(
+ "Epilogue tail-folding is not supported with interleaved accesses "
+ "when masking them isn't supported",
+ "InvalidTailFoldedEpilogue", ORE, L);
+ return CM_EpilogueAllowed;
+ }
+
if (ForcePartialAliasingVectorization) {
reportVectorizationInfo(
"Epilogue tail-folding is not supported with alias masking",
@@ -7886,7 +7897,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Config, IAI, PSE, ORE, GetBPI);
EpilogueLowering EpilogueTailLoweringStatus =
- getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints);
+ getEpilogueTailLowering(LVP.getCostModel(), L, ORE, LVL, Hints, TTI);
if (EpilogueTailLoweringStatus ==
EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
// TODO: Apply tail-folding on the vectorized epilogue loop.
diff --git a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
index 0fff5b9e889ad..d6ae51e3902fe 100644
--- a/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/fold-epilogue-tail.ll
@@ -23,6 +23,9 @@
; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 -force-partial-aliasing-vectorization \
; RUN: -force-target-supports-masked-memory-ops < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ALIAS-MASK
+; RUN: %{cmd} -force-vector-width=16 -epilogue-vectorization-force-VF=8 \
+; RUN: -enable-interleaved-mem-accesses=true < %s 2>&1 | FileCheck %s --check-prefix=CHECK-INVALID-INTERLEAVE
+
define void @test_epilogue_tf(ptr %A, i64 %n, i8 %val) {
; CHECK-LABEL: LV: Checking a loop in 'test_epilogue_tf'
; CHECK: LV: epilogue tail-folding is not supported yet
@@ -217,6 +220,36 @@ exit:
ret void
}
+ at AB = common global [1024 x i32] zeroinitializer, align 4
+ at CD = common global [1024 x i32] zeroinitializer, align 4
+define void @test_no_masked_interleave_support(i32 %C, i32 %D) {
+; CHECK-INVALID-INTERLEAVE-LABEL: LV: Checking a loop in 'test_no_masked_interleave_support'
+; CHECK-INVALID-INTERLEAVE: remark: <unknown>:0:0: Epilogue tail-folding is not supported with interleaved accesses when masking them isn't supported
+;
+entry:
+ br label %for.body
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx0 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %indvars.iv
+ %tmp = load i32, ptr %arrayidx0, align 4
+ %tmp1 = or disjoint i64 %indvars.iv, 1
+ %arrayidx1 = getelementptr inbounds [1024 x i32], ptr @AB, i64 0, i64 %tmp1
+ %tmp2 = load i32, ptr %arrayidx1, align 4
+ %add = add nsw i32 %tmp, %C
+ %mul = mul nsw i32 %tmp2, %D
+ %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %indvars.iv
+ store i32 %add, ptr %arrayidx2, align 4
+ %arrayidx3 = getelementptr inbounds [1024 x i32], ptr @CD, i64 0, i64 %tmp1
+ store i32 %mul, ptr %arrayidx3, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+ %cmp = icmp slt i64 %indvars.iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
define void @test_outer_loop(ptr %A, i64 %m) {
; CHECK-OUTER-LOOP-LABEL: Checking a loop in 'test_outer_loop'
; CHECK-OUTER-LOOP: remark: <unknown>:0:0: Epilogue tail-folding is not supported for outer loop
More information about the llvm-commits
mailing list