[llvm] [VPlan] Check FOR/FMinMaxNum epilogue restrictions in VPlan. (PR #191815)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 03:54:46 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/191815
>From 9acff8b0f967c5f973be6e455069ba39d5e55c12 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 11 Apr 2026 23:27:42 +0100
Subject: [PATCH 1/2] [VPlan] Check FOR/FMinMaxNum epilogue restrictions in
VPlan.
Move checking of FOR/FMinMaxNum restriction checks for epilogue
vectorization to hasUnsupportedHeaderPhiRecipe and perform checks
directly on VPlan.
This unifies the checking code and enables epilogue vectorization of
VPlans with dead FORs, although the latter should be cleaned up by
scalar optimizations earlier in practice.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 29 +++---
...g-vectorization-fixed-order-recurrences.ll | 95 ++++++++++++++-----
2 files changed, 84 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a7bbd94a0c776..043690170a732 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4267,13 +4267,20 @@ static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
return any_of(
Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
[](VPRecipeBase &R) {
+ // Fixed-order recurrences need special handling and are currently
+ // unsupported.
+ if (isa<VPFirstOrderRecurrencePHIRecipe>(R))
+ return true;
if (auto *WidenInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
return !WidenInd->getPHINode();
auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(&R);
if (!RedPhi)
return false;
- if (RecurrenceDescriptor::isFindLastRecurrenceKind(
- RedPhi->getRecurrenceKind()) ||
+ // FMinNum/FMaxNum, FindLast reductions, and reductions without
+ // underlying value need special handling and are currently unsupported.
+ RecurKind Kind = RedPhi->getRecurrenceKind();
+ if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
+ RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
!RedPhi->getUnderlyingValue())
return true;
// FindIV reductions with sunk expressions are not yet supported for
@@ -4281,8 +4288,7 @@ static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
// expression domain (e.g., mul(ReducedIV, 3)), but the epilogue tracks
// raw IV values. A sunk expression is identified by a non-VPInstruction
// user of ComputeReductionResult.
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(
- RedPhi->getRecurrenceKind())) {
+ if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
assert(RdxResult &&
"FindIV reduction must have ComputeReductionResult");
@@ -4295,19 +4301,8 @@ static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
VPlan &MainPlan) const {
- // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
- // reductions need special handling and are currently unsupported.
- if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
- if (!Legal->isReductionVariable(&Phi))
- return Legal->isFixedOrderRecurrence(&Phi);
- RecurKind Kind =
- Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
- return RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind);
- }))
- return false;
-
- // FindLast reductions and inductions without underlying PHI require special
- // handling and are currently not supported for epilogue vectorization.
+ // Bail out if the plan contains header phi recipes not yet supported
+ // for epilogue vectorization.
if (hasUnsupportedHeaderPhiRecipe(MainPlan))
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
index 4a1472783a00c..d778d1e67da54 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
@@ -6,6 +6,9 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK-LABEL: define void @dead_for(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -24,21 +27,42 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i32 7
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[WIDE_LOAD5]], splat (i64 10)
+; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i32 3
+; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N8]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[SCALAR_RECUR_INIT9:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT9]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[L]] = load i64, ptr [[GEP1]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
-; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -79,7 +103,7 @@ define i64 @for_phi_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i32 6
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i32 7
@@ -98,7 +122,7 @@ define i64 @for_phi_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store i64 [[FOR]], ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
@@ -128,6 +152,9 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-LABEL: define i64 @for_phi_not_used_in_loop_and_live_out(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -142,29 +169,51 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i32 6
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i32 7
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[WIDE_LOAD5]], splat (i64 10)
+; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i32 2
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i32 3
+; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[L]] = load i64, ptr [[GEP1]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
-; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
; CHECK-NEXT: ret i64 [[RES]]
;
>From 0d92c416abf3fc475a197f00a5b13c05d3081c94 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 23 Apr 2026 20:15:10 +0100
Subject: [PATCH 2/2] !fixup address comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 55 ++++++++++---------
...g-vectorization-fixed-order-recurrences.ll | 6 +-
2 files changed, 32 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7f4b581d1414e..52afa5e450a2f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3512,35 +3512,38 @@ static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
return any_of(
Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
[](VPRecipeBase &R) {
- // Fixed-order recurrences need special handling and are currently
- // unsupported.
- if (isa<VPFirstOrderRecurrencePHIRecipe>(R))
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
+ // TODO: Add support for fixed-order recurrences.
return true;
- if (auto *WidenInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
- return !WidenInd->getPHINode();
- auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!RedPhi)
+ case VPRecipeBase::VPWidenIntOrFpInductionSC:
+ return !cast<VPWidenIntOrFpInductionRecipe>(&R)->getPHINode();
+ case VPRecipeBase::VPReductionPHISC: {
+ auto *RedPhi = cast<VPReductionPHIRecipe>(&R);
+ // TODO: Support FMinNum/FMaxNum, FindLast reductions, and reductions
+ // without underlying values.
+ RecurKind Kind = RedPhi->getRecurrenceKind();
+ if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
+ RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
+ !RedPhi->getUnderlyingValue())
+ return true;
+ // TODO: Add support for FindIV reductions with sunk expressions: the
+ // resume value from the main loop is in expression domain (e.g.,
+ // mul(ReducedIV, 3)), but the epilogue tracks raw IV values. A sunk
+ // expression is identified by a non-VPInstruction user of
+ // ComputeReductionResult.
+ if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
+ auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
+ assert(RdxResult &&
+ "FindIV reduction must have ComputeReductionResult");
+ return any_of(RdxResult->users(),
+ std::not_fn(IsaPred<VPInstruction>));
+ }
return false;
- // FMinNum/FMaxNum, FindLast reductions, and reductions without
- // underlying value need special handling and are currently unsupported.
- RecurKind Kind = RedPhi->getRecurrenceKind();
- if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
- RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
- !RedPhi->getUnderlyingValue())
- return true;
- // FindIV reductions with sunk expressions are not yet supported for
- // epilogue vectorization: the resume value from the main loop is in
- // expression domain (e.g., mul(ReducedIV, 3)), but the epilogue tracks
- // raw IV values. A sunk expression is identified by a non-VPInstruction
- // user of ComputeReductionResult.
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
- auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
- assert(RdxResult &&
- "FindIV reduction must have ComputeReductionResult");
- return any_of(RdxResult->users(),
- std::not_fn(IsaPred<VPInstruction>));
}
- return false;
+ default:
+ return false;
+ };
});
}
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
index f14a59d363dc2..370051d8f216e 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
@@ -46,7 +46,7 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i32 3
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 3
; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[CMP_N8]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
@@ -193,8 +193,8 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i32 2
-; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i32 3
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 2
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 3
; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
More information about the llvm-commits
mailing list