[llvm] 7d0308c - [VPlan] Check FOR/FMinMaxNum epilogue restrictions in VPlan. (#191815)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 04:27:53 PDT 2026
Author: Florian Hahn
Date: 2026-04-29T11:27:49Z
New Revision: 7d0308c538c8f732f3293aea6694c7c3018ddc8c
URL: https://github.com/llvm/llvm-project/commit/7d0308c538c8f732f3293aea6694c7c3018ddc8c
DIFF: https://github.com/llvm/llvm-project/commit/7d0308c538c8f732f3293aea6694c7c3018ddc8c.diff
LOG: [VPlan] Check FOR/FMinMaxNum epilogue restrictions in VPlan. (#191815)
Move checking of FOR/FMinMaxNum restriction checks for epilogue
vectorization to hasUnsupportedHeaderPhiRecipe and perform checks
directly on VPlan.
This unifies the checking code and enables epilogue vectorization of
VPlans with dead FORs, although the latter should be cleaned up by
scalar optimizations earlier in practice.
PR: https://github.com/llvm/llvm-project/pull/191815
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6e7a17c8b59d5..7bff67d474181 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3512,47 +3512,45 @@ static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
return any_of(
Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
[](VPRecipeBase &R) {
- if (auto *WidenInd = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
- return !WidenInd->getPHINode();
- auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!RedPhi)
- return false;
- if (RecurrenceDescriptor::isFindLastRecurrenceKind(
- RedPhi->getRecurrenceKind()) ||
- !RedPhi->getUnderlyingValue())
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
+ // TODO: Add support for fixed-order recurrences.
return true;
- // FindIV reductions with sunk expressions are not yet supported for
- // epilogue vectorization: the resume value from the main loop is in
- // expression domain (e.g., mul(ReducedIV, 3)), but the epilogue tracks
- // raw IV values. A sunk expression is identified by a non-VPInstruction
- // user of ComputeReductionResult.
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(
- RedPhi->getRecurrenceKind())) {
- auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
- assert(RdxResult &&
- "FindIV reduction must have ComputeReductionResult");
- return any_of(RdxResult->users(),
- std::not_fn(IsaPred<VPInstruction>));
+ case VPRecipeBase::VPWidenIntOrFpInductionSC:
+ return !cast<VPWidenIntOrFpInductionRecipe>(&R)->getPHINode();
+ case VPRecipeBase::VPReductionPHISC: {
+ auto *RedPhi = cast<VPReductionPHIRecipe>(&R);
+ // TODO: Support FMinNum/FMaxNum, FindLast reductions, and reductions
+ // without underlying values.
+ RecurKind Kind = RedPhi->getRecurrenceKind();
+ if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
+ RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
+ !RedPhi->getUnderlyingValue())
+ return true;
+ // TODO: Add support for FindIV reductions with sunk expressions: the
+ // resume value from the main loop is in expression domain (e.g.,
+ // mul(ReducedIV, 3)), but the epilogue tracks raw IV values. A sunk
+ // expression is identified by a non-VPInstruction user of
+ // ComputeReductionResult.
+ if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
+ auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
+ assert(RdxResult &&
+ "FindIV reduction must have ComputeReductionResult");
+ return any_of(RdxResult->users(),
+ std::not_fn(IsaPred<VPInstruction>));
+ }
+ return false;
}
- return false;
+ default:
+ return false;
+ };
});
}
bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
VPlan &MainPlan) const {
- // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
- // reductions need special handling and are currently unsupported.
- if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
- if (!Legal->isReductionVariable(&Phi))
- return Legal->isFixedOrderRecurrence(&Phi);
- RecurKind Kind =
- Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
- return RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind);
- }))
- return false;
-
- // FindLast reductions and inductions without underlying PHI require special
- // handling and are currently not supported for epilogue vectorization.
+ // Bail out if the plan contains header phi recipes not yet supported
+ // for epilogue vectorization.
if (hasUnsupportedHeaderPhiRecipe(MainPlan))
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
index 4cabbf0e9f728..370051d8f216e 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-fixed-order-recurrences.ll
@@ -6,6 +6,9 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK-LABEL: define void @dead_for(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -24,21 +27,42 @@ define void @dead_for(ptr %a, i64 %N) {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[WIDE_LOAD5]], splat (i64 10)
+; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 3
+; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N8]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[SCALAR_RECUR_INIT9:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT9]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[L]] = load i64, ptr [[GEP1]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
-; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -79,7 +103,7 @@ define i64 @for_phi_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 6
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7
@@ -98,7 +122,7 @@ define i64 @for_phi_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store i64 [[FOR]], ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
@@ -128,6 +152,9 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-LABEL: define i64 @for_phi_not_used_in_loop_and_live_out(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -142,29 +169,51 @@ define i64 @for_phi_not_used_in_loop_and_live_out(ptr %a, i64 %N) {
; CHECK-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 6
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK1]] ]
+; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[VEC_EPILOG_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[L]] = load i64, ptr [[GEP]], align 4
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[GEP]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[WIDE_LOAD5]], splat (i64 10)
+; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 2
+; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD5]], i64 3
+; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[L]] = load i64, ptr [[GEP1]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add i64 [[L]], 10
-; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[FOR_LCSSA:%.*]] = phi i64 [ [[FOR]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[L_LCSSA:%.*]] = phi i64 [ [[L]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[VECTOR_RECUR_EXTRACT8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RES:%.*]] = add i64 [[L_LCSSA]], [[FOR_LCSSA]]
; CHECK-NEXT: ret i64 [[RES]]
;
More information about the llvm-commits
mailing list