[llvm] d478502 - [VPlan] Ensure that IV resume phi for epilogue is always first. (NFCI)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 5 13:07:29 PDT 2025
Author: Florian Hahn
Date: 2025-08-05T21:06:41+01:00
New Revision: d478502a420c75ca6c52205a58e465998a249826
URL: https://github.com/llvm/llvm-project/commit/d478502a420c75ca6c52205a58e465998a249826
DIFF: https://github.com/llvm/llvm-project/commit/d478502a420c75ca6c52205a58e465998a249826.diff
LOG: [VPlan] Ensure that IV resume phi for epilogue is always first. (NFCI)
Update handling of canonical IV resume phi for the epilogue loop to make
sure the resume phi for the canonical IV is always the first phi in the
scalar preheader.
This makes it easier to retrieve it in preparePlanForEpilogueVectorLoop.
For now, we keep an assert to make sure we use the same resume phi as
before. This will be removed in the future.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index eb0e0fd7b3d8e..a52aa8420b301 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9782,17 +9782,26 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
VPValue *VectorTC = &MainPlan.getVectorTripCount();
// If there is a suitable resume value for the canonical induction in the
- // scalar (which will become vector) epilogue loop we are done. Otherwise
- // create it below.
- if (any_of(*MainScalarPH, [VectorTC](VPRecipeBase &R) {
+ // scalar (which will become vector) epilogue loop, use it and move it to the
+ // beginning of the scalar preheader. Otherwise create it below.
+ auto ResumePhiIter =
+ find_if(MainScalarPH->phis(), [VectorTC](VPRecipeBase &R) {
return match(&R, m_VPInstruction<Instruction::PHI>(m_Specific(VectorTC),
m_SpecificInt(0)));
- }))
- return;
- VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
- ScalarPHBuilder.createScalarPhi(
- {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
- "vec.epilog.resume.val");
+ });
+ VPPhi *ResumePhi = nullptr;
+ if (ResumePhiIter == MainScalarPH->phis().end()) {
+ VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
+ ResumePhi = ScalarPHBuilder.createScalarPhi(
+ {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
+ "vec.epilog.resume.val");
+ } else {
+ ResumePhi = cast<VPPhi>(&*ResumePhiIter);
+ if (MainScalarPH->begin() == MainScalarPH->end())
+ ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->end());
+ else if (&*MainScalarPH->begin() != ResumePhi)
+ ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->begin());
+ }
}
/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
@@ -9813,30 +9822,35 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
// When vectorizing the epilogue loop, the canonical induction start
// value needs to be changed from zero to the value after the main
// vector loop. Find the resume value created during execution of the main
- // VPlan.
+ // VPlan. It must be the first phi in the loop preheader.
// FIXME: Improve modeling for canonical IV start values in the epilogue
// loop.
using namespace llvm::PatternMatch;
- Type *IdxTy = IV->getScalarType();
- PHINode *EPResumeVal = find_singleton<PHINode>(
- L->getLoopPreheader()->phis(),
- [&EPI, IdxTy](PHINode &P, bool) -> PHINode * {
- if (P.getType() == IdxTy &&
- match(
- P.getIncomingValueForBlock(EPI.MainLoopIterationCountCheck),
- m_SpecificInt(0)) &&
- any_of(P.incoming_values(),
- [&EPI](Value *Inc) {
- return Inc == EPI.VectorTripCount;
- }) &&
- all_of(P.incoming_values(), [&EPI](Value *Inc) {
- return Inc == EPI.VectorTripCount ||
- match(Inc, m_SpecificInt(0));
- }))
- return &P;
- return nullptr;
- });
- assert(EPResumeVal && "must have a resume value for the canonical IV");
+ PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
+ assert(EPResumeVal->getType() == IV->getScalarType() &&
+ match(EPResumeVal->getIncomingValueForBlock(
+ EPI.MainLoopIterationCountCheck),
+ m_SpecificInt(0)) &&
+ EPResumeVal ==
+ find_singleton<PHINode>(
+ L->getLoopPreheader()->phis(),
+ [&EPI, IV](PHINode &P, bool) -> PHINode * {
+ if (P.getType() == IV->getScalarType() &&
+ match(P.getIncomingValueForBlock(
+ EPI.MainLoopIterationCountCheck),
+ m_SpecificInt(0)) &&
+ any_of(P.incoming_values(),
+ [&EPI](Value *Inc) {
+ return Inc == EPI.VectorTripCount;
+ }) &&
+ all_of(P.incoming_values(), [&EPI](Value *Inc) {
+ return Inc == EPI.VectorTripCount ||
+ match(Inc, m_SpecificInt(0));
+ }))
+ return &P;
+ return nullptr;
+ }) &&
+ "Epilogue resume phis do not match!");
VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
assert(all_of(IV->users(),
[](const VPUser *U) {
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 289e8c5b6385b..5e97cedb452b4 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -606,8 +606,8 @@ define i64 @test_reduction_with_widen_induction_order_2(ptr %A, i64 %N) {
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
More information about the llvm-commits
mailing list