[llvm] [VPlan] Support plain CFG plans in the verifier. (PR #181817)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 19 01:46:01 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/181817
>From eeed4ff248aa3c3badac30934bee1391932b918e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 16 Feb 2026 14:26:40 +0000
Subject: [PATCH 1/2] [VPlan] Support plain CFG plans in the verifier.
Update the verifier to support verifying plain CFG plans in the verifier
and add missing support for switch opcodes in type analysis.
This allows the verifier to handle early plain-CFG plans, allowing using
RUN_PASS wrapper for early transforms.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 36 +++++------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 1 +
.../Vectorize/VPlanConstruction.cpp | 10 ++++
.../Transforms/Vectorize/VPlanVerifier.cpp | 60 +++++++++++++++----
.../VPlan/vplan-print-after-all.ll | 7 +++
5 files changed, 86 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 762f355fb742a..dc6267a73a183 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7681,12 +7681,12 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);
// Create recipes for header phis.
- VPlanTransforms::createHeaderPhiRecipes(
- *VPlan0, PSE, *OrigLoop, Legal->getInductionVars(),
- Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
- CM.getInLoopReductions(), Hints.allowReordering());
+ RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
+ *OrigLoop, Legal->getInductionVars(),
+ Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
+ CM.getInLoopReductions(), Hints.allowReordering());
- VPlanTransforms::simplifyRecipes(*VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
// If we're vectorizing a loop with an uncountable exit, make sure that the
// recipes are safe to handle.
// TODO: Remove this once we can properly check the VPlan itself for both
@@ -7698,15 +7698,16 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
? UncountableExitStyle::MaskedHandleExitInScalarLoop
: UncountableExitStyle::ReadOnly;
- if (!VPlanTransforms::handleEarlyExits(*VPlan0, EEStyle, OrigLoop, PSE, *DT,
- Legal->getAssumptionCache()))
+ if (!RUN_VPLAN_PASS(VPlanTransforms::handleEarlyExits, *VPlan0, EEStyle,
+ OrigLoop, PSE, *DT, Legal->getAssumptionCache()))
return;
- VPlanTransforms::addMiddleCheck(*VPlan0, CM.foldTailByMasking());
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *VPlan0);
+
+ RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0,
+ CM.foldTailByMasking());
+ RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0);
if (CM.foldTailByMasking())
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *VPlan0);
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::introduceMasksAndLinearize,
- *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
@@ -7831,8 +7832,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (CM.blockNeedsPredicationForAnyReason(BB))
BlocksNeedingPredication.insert(BB);
- VPlanTransforms::createInLoopReductionRecipes(*Plan, BlocksNeedingPredication,
- Range.Start);
+ RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
+ BlocksNeedingPredication, Range.Start);
VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind, CM.PSE,
OrigLoop);
@@ -7891,9 +7892,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
- // TODO: We can't call runPass on these transforms yet, due to verifier
- // failures.
- VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range);
+ RUN_VPLAN_PASS(VPlanTransforms::addExitUsersForFirstOrderRecurrences, *Plan,
+ Range);
// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -7922,7 +7922,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
return nullptr;
- VPlanTransforms::removeBranchOnConst(*Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst,*Plan, false);
// Create partial reduction recipes for scaled reductions and transform
// recipes to abstract recipes if it is legal and beneficial and clamp the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 0af8886281f8a..3af1cae04a15c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -137,6 +137,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::BranchOnTwoConds:
case VPInstruction::BranchOnCount:
case Instruction::Store:
+ case Instruction::Switch:
return Type::getVoidTy(Ctx);
case Instruction::Load:
return cast<LoadInst>(R->getUnderlyingValue())->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index af0882dfbad8f..489de6accaec6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -930,6 +930,16 @@ void VPlanTransforms::createInLoopReductionRecipes(
LinkVPBB->appendRecipe(RedRecipe);
CurrentLink->replaceAllUsesWith(RedRecipe);
+ // Move any store recipes using the RedRecipe that appear before it in the
+ // same block to just after the RedRecipe.
+ for (VPUser *U : make_early_inc_range(RedRecipe->users())) {
+ auto *UserR = dyn_cast<VPRecipeBase>(U);
+ if (!UserR || UserR->getParent() != LinkVPBB)
+ continue;
+ if (!match(UserR, m_VPInstruction<Instruction::Store>()))
+ continue;
+ UserR->moveAfter(RedRecipe);
+ }
ToDelete.push_back(CurrentLink);
PreviousLink = RedRecipe;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index d1faccb3995b9..9187f4c1665a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -228,6 +228,11 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
return false;
}
+ // MaskedCond may be used from blocks it don't dominate; the block will be
+ // linearized and it will dominate its users after linearization.
+ if (match(&R, m_VPInstruction<VPInstruction::MaskedCond>()))
+ continue;
+
for (const VPUser *U : V->users()) {
auto *UI = cast<VPRecipeBase>(U);
if (isa<VPIRPhi>(UI) &&
@@ -272,6 +277,19 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
continue;
}
+ // Recipes in blocks with a MaskedCond may be used in exit blocks; the
+ // block will be linearized and its recipes will dominate their users
+ // after linearization.
+ bool BlockHasMaskedCond = any_of(*VPBB, [](const VPRecipeBase &R) {
+ return match(&R, m_VPInstruction<VPInstruction::MaskedCond>());
+ });
+ if (BlockHasMaskedCond &&
+ any_of(VPBB->getPlan()->getExitBlocks(), [UI](VPIRBasicBlock *EB) {
+ return is_contained(EB->getPredecessors(), UI->getParent());
+ })) {
+ continue;
+ }
+
errs() << "Use before def!\n";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
VPSlotTracker Tracker(VPBB->getPlan());
@@ -318,9 +336,38 @@ bool VPlanVerifier::verifyBlock(const VPBlockBase *VPB) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
// Check block's condition bit.
if (VPBB && !isa<VPIRBasicBlock>(VPB)) {
- if (VPB->getNumSuccessors() > 1 ||
- (VPBB->getParent() && VPBB->isExiting() &&
- !VPBB->getParent()->isReplicator())) {
+ // For plain CFG VPlans, verify header and latch block structure.
+ if (!VPBB->getParent()) {
+ if (VPBlockUtils::isHeader(VPBB, VPDT)) {
+ if (VPB->getNumPredecessors() != 2) {
+ errs()
+ << "Header block in plain CFG VPlan must have 2 predecessors!\n";
+ return false;
+ }
+ // Predecessor 0 is preheader, predecessor 1 is latch.
+ if (!VPBlockUtils::isLatch(VPB->getPredecessors()[1], VPDT)) {
+ errs() << "Header's second predecessor must be the latch!\n";
+ return false;
+ }
+ }
+
+ if (VPBlockUtils::isLatch(VPBB, VPDT)) {
+ auto BranchTerminator =
+ m_CombineOr(m_BranchOnCond(),
+ m_CombineOr(m_BranchOnCount(), m_BranchOnTwoConds()));
+ if (!match(VPBB->getTerminator(), BranchTerminator)) {
+ errs() << "Latch block must have a branch terminator!\n";
+ return false;
+ }
+ // Successor 0 is middle block, successor 1 is header.
+ if (VPBlockUtils::isHeader(VPB->getSuccessors()[0], VPDT)) {
+ errs() << "Latch's first successor must not be the header (must be "
+ "middle block)!\n";
+ return false;
+ }
+ }
+ } else if (VPB->getNumSuccessors() > 1 ||
+ (VPBB->isExiting() && !VPBB->getParent()->isReplicator())) {
if (!VPBB->getTerminator()) {
errs() << "Block has multiple successors but doesn't "
"have a proper branch recipe!\n";
@@ -360,13 +407,6 @@ bool VPlanVerifier::verifyBlock(const VPBlockBase *VPB) {
return false;
}
}
- // For plain CFG VPlans (no parent region), verify header/latch structure.
- if (VPBB && !VPBB->getParent() && VPBlockUtils::isHeader(VPBB, VPDT) &&
- !VPBlockUtils::isLatch(VPB->getPredecessors()[1], VPDT)) {
- errs() << "Header's second predecessor must be the latch!\n";
- return false;
- }
-
return !VPBB || verifyVPBasicBlock(VPBB);
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 8d7c014b302a3..847e849fa2ffd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -4,14 +4,21 @@
; Verify that `-vplan-print-after-all` option works.
; CHECK: VPlan for loop in 'foo' after printAfterInitialConstruction
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::simplifyRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleEarlyExits
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addMiddleCheck
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInLoopReductionRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addExitUsersForFirstOrderRecurrences
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleMultiUseReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleMaxMinNumReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleFindLastReductions
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::removeBranchOnConst
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createPartialReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::convertToAbstractRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInterleaveGroups
>From fd4ded976875637a5200d85631b44bf553b7b767 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 28 Feb 2026 07:41:14 +0000
Subject: [PATCH 2/2] Enable
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dc6267a73a183..3c544f291dce7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -361,7 +361,7 @@ cl::opt<bool>
#ifdef EXPENSIVE_CHECKS
cl::init(true),
#else
- cl::init(false),
+ cl::init(true),
#endif
cl::Hidden,
cl::desc("Verify VPlans after VPlan transforms."));
More information about the llvm-commits
mailing list