[llvm] 9f2035d - [VPlan] Move tail folding logic out of addMiddleCheck. NFC (#203475)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 14 19:50:08 PDT 2026
Author: Luke Lau
Date: 2026-06-15T02:50:03Z
New Revision: 9f2035d27cb534ab7be6fe3e41603eba74e66df5
URL: https://github.com/llvm/llvm-project/commit/9f2035d27cb534ab7be6fe3e41603eba74e66df5
DIFF: https://github.com/llvm/llvm-project/commit/9f2035d27cb534ab7be6fe3e41603eba74e66df5.diff
LOG: [VPlan] Move tail folding logic out of addMiddleCheck. NFC (#203475)
We simplify the TripCount == VectorTrip count condition with tail
folding, but we can just do that in foldTailByMasking and keep the
logic in one place instead.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 70ef858292a69..cd97a7055fc97 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6569,8 +6569,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
// loop with an in-loop mask, then the middle check has already been
// created to compare against the actual number of lanes executed.
if (EEStyle != UncountableExitStyle::MaskedHandleExitInScalarLoop)
- RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0,
- CM.foldTailByMasking());
+ RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
if (CM.foldTailByMasking())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8766055c6bc89..8aa7b01e8d58b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1301,7 +1301,7 @@ bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
return true;
}
-void VPlanTransforms::addMiddleCheck(VPlan &Plan, bool TailFolded) {
+void VPlanTransforms::addMiddleCheck(VPlan &Plan) {
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
// If MiddleVPBB has a single successor then the original loop does not exit
// via the latch and the single successor must be the scalar preheader.
@@ -1332,12 +1332,9 @@ void VPlanTransforms::addMiddleCheck(VPlan &Plan, bool TailFolded) {
auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
DebugLoc LatchDL = LatchVPBB->getTerminator()->getDebugLoc();
VPBuilder Builder(MiddleVPBB);
- VPValue *Cmp;
- if (TailFolded)
- Cmp = Plan.getTrue();
- else
- Cmp = Builder.createICmp(CmpInst::ICMP_EQ, Plan.getTripCount(),
- &Plan.getVectorTripCount(), LatchDL, "cmp.n");
+ VPValue *Cmp =
+ Builder.createICmp(CmpInst::ICMP_EQ, Plan.getTripCount(),
+ &Plan.getVectorTripCount(), LatchDL, "cmp.n");
Builder.createNaryOp(VPInstruction::BranchOnCond, {Cmp}, LatchDL);
}
@@ -1449,6 +1446,14 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Builder.createNaryOp(VPInstruction::ExtractLane, {LastActiveLane, Op});
R.getVPSingleValue()->replaceAllUsesWith(Ext);
}
+
+ // VectorTripCount now equals TripCount so simplify the MiddleVPBB branch.
+ assert(match(Plan.getMiddleBlock()->getTerminator(),
+ m_BranchOnCond(m_SpecificICmp(
+ CmpInst::ICMP_EQ, m_Specific(Plan.getTripCount()),
+ m_Specific(&Plan.getVectorTripCount())))) &&
+ "Unexpected MiddleVPBB branch");
+ Plan.getMiddleBlock()->getTerminator()->setOperand(0, Plan.getTrue());
}
/// Insert \p CheckBlockVPBB on the edge leading to the vector preheader,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index a8c92e6d034fe..251edfba14b26 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -172,7 +172,7 @@ struct VPlanTransforms {
/// If a check is needed to guard executing the scalar epilogue loop, it will
/// be added to the middle block.
- LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan, bool TailFolded);
+ LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan);
// Create a check in \p CheckBlock to see if the vector loop should be
// executed. May create VPExpandSCEV recipes in the plan's entry block.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
index 464bc967bc65d..c5623e3de4b86 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -47,6 +47,7 @@ define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane vp<[[VP13]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
@@ -142,6 +143,7 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane vp<[[VP13]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
@@ -226,6 +228,7 @@ define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
@@ -305,6 +308,7 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-lane vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-lane vp<[[VP15]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index c38b4cc730f47..9420420c45517 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -98,7 +98,7 @@ class VPlanTestIRBase : public testing::Test {
}
VPlanTransforms::handleEarlyExits(*Plan, Style, L, PSE, *DT, AC.get());
- VPlanTransforms::addMiddleCheck(*Plan, false);
+ VPlanTransforms::addMiddleCheck(*Plan);
if (CreateLoopRegions)
VPlanTransforms::createLoopRegions(*Plan, {});
More information about the llvm-commits
mailing list