[llvm] 26b7fd5 - [VPlan] Support plain CFG plans in the verifier. (#181817)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 19 02:56:24 PDT 2026
Author: Florian Hahn
Date: 2026-04-19T10:56:21+01:00
New Revision: 26b7fd51095221e03ce2a2170b6d168652ba3c7d
URL: https://github.com/llvm/llvm-project/commit/26b7fd51095221e03ce2a2170b6d168652ba3c7d
DIFF: https://github.com/llvm/llvm-project/commit/26b7fd51095221e03ce2a2170b6d168652ba3c7d.diff
LOG: [VPlan] Support plain CFG plans in the verifier. (#181817)
Update the verifier to support verifying plain CFG plans in the verifier
and add missing support for switch opcodes in type analysis.
This allows the verifier to handle early plain-CFG plans, allowing using
RUN_PASS wrapper for early transforms.
PR: https://github.com/llvm/llvm-project/pull/181817
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 762f355fb742a..e17a5b5434664 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7681,12 +7681,12 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);
// Create recipes for header phis.
- VPlanTransforms::createHeaderPhiRecipes(
- *VPlan0, PSE, *OrigLoop, Legal->getInductionVars(),
- Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
- CM.getInLoopReductions(), Hints.allowReordering());
+ RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
+ *OrigLoop, Legal->getInductionVars(),
+ Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
+ CM.getInLoopReductions(), Hints.allowReordering());
- VPlanTransforms::simplifyRecipes(*VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
// If we're vectorizing a loop with an uncountable exit, make sure that the
// recipes are safe to handle.
// TODO: Remove this once we can properly check the VPlan itself for both
@@ -7698,15 +7698,16 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
? UncountableExitStyle::MaskedHandleExitInScalarLoop
: UncountableExitStyle::ReadOnly;
- if (!VPlanTransforms::handleEarlyExits(*VPlan0, EEStyle, OrigLoop, PSE, *DT,
- Legal->getAssumptionCache()))
+ if (!RUN_VPLAN_PASS(VPlanTransforms::handleEarlyExits, *VPlan0, EEStyle,
+ OrigLoop, PSE, *DT, Legal->getAssumptionCache()))
return;
- VPlanTransforms::addMiddleCheck(*VPlan0, CM.foldTailByMasking());
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *VPlan0);
+
+ RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0,
+ CM.foldTailByMasking());
+ RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0);
if (CM.foldTailByMasking())
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *VPlan0);
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::introduceMasksAndLinearize,
- *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
@@ -7831,8 +7832,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (CM.blockNeedsPredicationForAnyReason(BB))
BlocksNeedingPredication.insert(BB);
- VPlanTransforms::createInLoopReductionRecipes(*Plan, BlocksNeedingPredication,
- Range.Start);
+ RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
+ BlocksNeedingPredication, Range.Start);
VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind, CM.PSE,
OrigLoop);
@@ -7891,9 +7892,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
- // TODO: We can't call runPass on these transforms yet, due to verifier
- // failures.
- VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range);
+ RUN_VPLAN_PASS(VPlanTransforms::addExitUsersForFirstOrderRecurrences, *Plan,
+ Range);
// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -7922,7 +7922,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
return nullptr;
- VPlanTransforms::removeBranchOnConst(*Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, *Plan, false);
// Create partial reduction recipes for scaled reductions and transform
// recipes to abstract recipes if it is legal and beneficial and clamp the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 0af8886281f8a..3af1cae04a15c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -137,6 +137,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::BranchOnTwoConds:
case VPInstruction::BranchOnCount:
case Instruction::Store:
+ case Instruction::Switch:
return Type::getVoidTy(Ctx);
case Instruction::Load:
return cast<LoadInst>(R->getUnderlyingValue())->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index af0882dfbad8f..489de6accaec6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -930,6 +930,16 @@ void VPlanTransforms::createInLoopReductionRecipes(
LinkVPBB->appendRecipe(RedRecipe);
CurrentLink->replaceAllUsesWith(RedRecipe);
+ // Move any store recipes using the RedRecipe that appear before it in the
+ // same block to just after the RedRecipe.
+ for (VPUser *U : make_early_inc_range(RedRecipe->users())) {
+ auto *UserR = dyn_cast<VPRecipeBase>(U);
+ if (!UserR || UserR->getParent() != LinkVPBB)
+ continue;
+ if (!match(UserR, m_VPInstruction<Instruction::Store>()))
+ continue;
+ UserR->moveAfter(RedRecipe);
+ }
ToDelete.push_back(CurrentLink);
PreviousLink = RedRecipe;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index d1faccb3995b9..9187f4c1665a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -228,6 +228,11 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
return false;
}
+ // MaskedCond may be used from blocks it don't dominate; the block will be
+ // linearized and it will dominate its users after linearization.
+ if (match(&R, m_VPInstruction<VPInstruction::MaskedCond>()))
+ continue;
+
for (const VPUser *U : V->users()) {
auto *UI = cast<VPRecipeBase>(U);
if (isa<VPIRPhi>(UI) &&
@@ -272,6 +277,19 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
continue;
}
+ // Recipes in blocks with a MaskedCond may be used in exit blocks; the
+ // block will be linearized and its recipes will dominate their users
+ // after linearization.
+ bool BlockHasMaskedCond = any_of(*VPBB, [](const VPRecipeBase &R) {
+ return match(&R, m_VPInstruction<VPInstruction::MaskedCond>());
+ });
+ if (BlockHasMaskedCond &&
+ any_of(VPBB->getPlan()->getExitBlocks(), [UI](VPIRBasicBlock *EB) {
+ return is_contained(EB->getPredecessors(), UI->getParent());
+ })) {
+ continue;
+ }
+
errs() << "Use before def!\n";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
VPSlotTracker Tracker(VPBB->getPlan());
@@ -318,9 +336,38 @@ bool VPlanVerifier::verifyBlock(const VPBlockBase *VPB) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
// Check block's condition bit.
if (VPBB && !isa<VPIRBasicBlock>(VPB)) {
- if (VPB->getNumSuccessors() > 1 ||
- (VPBB->getParent() && VPBB->isExiting() &&
- !VPBB->getParent()->isReplicator())) {
+ // For plain CFG VPlans, verify header and latch block structure.
+ if (!VPBB->getParent()) {
+ if (VPBlockUtils::isHeader(VPBB, VPDT)) {
+ if (VPB->getNumPredecessors() != 2) {
+ errs()
+ << "Header block in plain CFG VPlan must have 2 predecessors!\n";
+ return false;
+ }
+ // Predecessor 0 is preheader, predecessor 1 is latch.
+ if (!VPBlockUtils::isLatch(VPB->getPredecessors()[1], VPDT)) {
+ errs() << "Header's second predecessor must be the latch!\n";
+ return false;
+ }
+ }
+
+ if (VPBlockUtils::isLatch(VPBB, VPDT)) {
+ auto BranchTerminator =
+ m_CombineOr(m_BranchOnCond(),
+ m_CombineOr(m_BranchOnCount(), m_BranchOnTwoConds()));
+ if (!match(VPBB->getTerminator(), BranchTerminator)) {
+ errs() << "Latch block must have a branch terminator!\n";
+ return false;
+ }
+ // Successor 0 is middle block, successor 1 is header.
+ if (VPBlockUtils::isHeader(VPB->getSuccessors()[0], VPDT)) {
+ errs() << "Latch's first successor must not be the header (must be "
+ "middle block)!\n";
+ return false;
+ }
+ }
+ } else if (VPB->getNumSuccessors() > 1 ||
+ (VPBB->isExiting() && !VPBB->getParent()->isReplicator())) {
if (!VPBB->getTerminator()) {
errs() << "Block has multiple successors but doesn't "
"have a proper branch recipe!\n";
@@ -360,13 +407,6 @@ bool VPlanVerifier::verifyBlock(const VPBlockBase *VPB) {
return false;
}
}
- // For plain CFG VPlans (no parent region), verify header/latch structure.
- if (VPBB && !VPBB->getParent() && VPBlockUtils::isHeader(VPBB, VPDT) &&
- !VPBlockUtils::isLatch(VPB->getPredecessors()[1], VPDT)) {
- errs() << "Header's second predecessor must be the latch!\n";
- return false;
- }
-
return !VPBB || verifyVPBasicBlock(VPBB);
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 8d7c014b302a3..847e849fa2ffd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -4,14 +4,21 @@
; Verify that `-vplan-print-after-all` option works.
; CHECK: VPlan for loop in 'foo' after printAfterInitialConstruction
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::simplifyRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleEarlyExits
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addMiddleCheck
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInLoopReductionRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addExitUsersForFirstOrderRecurrences
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleMultiUseReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleMaxMinNumReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleFindLastReductions
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::removeBranchOnConst
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createPartialReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::convertToAbstractRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInterleaveGroups
More information about the llvm-commits
mailing list