[llvm] e81fab6 - [VPlan] Verify final VPlan, just before execution. (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat May 17 09:53:30 PDT 2025
Author: Florian Hahn
Date: 2025-05-17T17:53:06+01:00
New Revision: e81fab68478754097c3354c0d2749ab009f8f0c8
URL: https://github.com/llvm/llvm-project/commit/e81fab68478754097c3354c0d2749ab009f8f0c8
DIFF: https://github.com/llvm/llvm-project/commit/e81fab68478754097c3354c0d2749ab009f8f0c8.diff
LOG: [VPlan] Verify final VPlan, just before execution. (NFC)
Add additional verifier call just before execution, to make sure the
final VPlan is valid.
Note that this currently requires disabling a small number of checks
when running late.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
llvm/lib/Transforms/Vectorize/VPlanVerifier.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d6af8a1435d07..148b187f4f25d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7802,7 +7802,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::removeDeadRecipes(BestVPlan);
VPlanTransforms::convertToConcreteRecipes(BestVPlan,
*Legal->getWidestInductionType());
-
// Perform the actual loop transformation.
VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
OrigLoop->getParentLoop(),
@@ -7843,6 +7842,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
if (VectorizingEpilogue)
VPlanTransforms::removeDeadRecipes(BestVPlan);
+ assert(verifyVPlanIsValid(BestVPlan, true /*VerifyLate*/) &&
+ "final VPlan is invalid");
+
ILV.printDebugTracesAtStart();
//===------------------------------------------------===//
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 75fc76321db2d..68b35d42e8674 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -28,6 +28,7 @@ namespace {
class VPlanVerifier {
const VPDominatorTree &VPDT;
VPTypeAnalysis &TypeInfo;
+ bool VerifyLate;
SmallPtrSet<BasicBlock *, 8> WrappedIRBBs;
@@ -60,8 +61,9 @@ class VPlanVerifier {
bool verifyRegionRec(const VPRegionBlock *Region);
public:
- VPlanVerifier(VPDominatorTree &VPDT, VPTypeAnalysis &TypeInfo)
- : VPDT(VPDT), TypeInfo(TypeInfo) {}
+ VPlanVerifier(VPDominatorTree &VPDT, VPTypeAnalysis &TypeInfo,
+ bool VerifyLate)
+ : VPDT(VPDT), TypeInfo(TypeInfo), VerifyLate(VerifyLate) {}
bool verify(const VPlan &Plan);
};
@@ -113,7 +115,7 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
RecipeI++;
}
- if (NumActiveLaneMaskPhiRecipes > 1) {
+ if (!VerifyLate && NumActiveLaneMaskPhiRecipes > 1) {
errs() << "There should be no more than one VPActiveLaneMaskPHIRecipe";
return false;
}
@@ -151,7 +153,7 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
}
return true;
};
- return all_of(EVL.users(), [&VerifyEVLUse](VPUser *U) {
+ return all_of(EVL.users(), [this, &VerifyEVLUse](VPUser *U) {
return TypeSwitch<const VPUser *, bool>(U)
.Case<VPWidenIntrinsicRecipe>([&](const VPWidenIntrinsicRecipe *S) {
return VerifyEVLUse(*S, S->getNumOperands() - 1);
@@ -174,7 +176,7 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
"users\n";
return false;
}
- if (!isa<VPEVLBasedIVPHIRecipe>(*I->users().begin())) {
+ if (!VerifyLate && !isa<VPEVLBasedIVPHIRecipe>(*I->users().begin())) {
errs() << "Result of VPInstruction::Add with EVL operand is "
"not used by VPEVLBasedIVPHIRecipe\n";
return false;
@@ -243,7 +245,9 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
continue;
}
// TODO: Also verify VPPredInstPHIRecipe.
- if (isa<VPPredInstPHIRecipe>(UI))
+ if (isa<VPPredInstPHIRecipe>(UI) ||
+ (isa<VPInstruction>(UI) && (cast<VPInstruction>(UI)->getOpcode() ==
+ VPInstruction::ResumePhi)))
continue;
// If the user is in the same block, check it comes after R in the
@@ -302,18 +306,20 @@ static bool hasDuplicates(const SmallVectorImpl<VPBlockBase *> &VPBlockVec) {
bool VPlanVerifier::verifyBlock(const VPBlockBase *VPB) {
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
// Check block's condition bit.
- if (VPB->getNumSuccessors() > 1 ||
- (VPBB && VPBB->getParent() && VPBB->isExiting() &&
- !VPBB->getParent()->isReplicator())) {
- if (!VPBB || !VPBB->getTerminator()) {
- errs() << "Block has multiple successors but doesn't "
- "have a proper branch recipe!\n";
- return false;
- }
- } else {
- if (VPBB && VPBB->getTerminator()) {
- errs() << "Unexpected branch recipe!\n";
- return false;
+ if (!isa<VPIRBasicBlock>(VPB)) {
+ if (VPB->getNumSuccessors() > 1 ||
+ (VPBB && VPBB->getParent() && VPBB->isExiting() &&
+ !VPBB->getParent()->isReplicator())) {
+ if (!VPBB || !VPBB->getTerminator()) {
+ errs() << "Block has multiple successors but doesn't "
+ "have a proper branch recipe!\n";
+ return false;
+ }
+ } else {
+ if (VPBB && VPBB->getTerminator()) {
+ errs() << "Unexpected branch recipe!\n";
+ return false;
+ }
}
}
@@ -409,6 +415,10 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
return false;
const VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
+ // TODO: Verify all blocks using vp_depth_first_deep iterators.
+ if (!TopRegion)
+ return true;
+
if (!verifyRegionRec(TopRegion))
return false;
@@ -423,7 +433,8 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
return false;
}
- if (!isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
+ // TODO: Remove once loop regions are dissolved before execution.
+ if (!VerifyLate && !isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
errs() << "VPlan vector loop header does not start with a "
"VPCanonicalIVPHIRecipe\n";
return false;
@@ -452,10 +463,10 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
return true;
}
-bool llvm::verifyVPlanIsValid(const VPlan &Plan) {
+bool llvm::verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate) {
VPDominatorTree VPDT;
VPDT.recalculate(const_cast<VPlan &>(Plan));
VPTypeAnalysis TypeInfo(Plan);
- VPlanVerifier Verifier(VPDT, TypeInfo);
+ VPlanVerifier Verifier(VPDT, TypeInfo, VerifyLate);
return Verifier.verify(Plan);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.h b/llvm/lib/Transforms/Vectorize/VPlanVerifier.h
index 3ddc49fda36b8..34a2fa2bab1e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.h
@@ -27,13 +27,15 @@
namespace llvm {
class VPlan;
-/// Verify invariants for general VPlans. Currently it checks the following:
+/// Verify invariants for general VPlans. If \p VerifyLate is passed, skip some
+/// checks that are not applicable at later stages of the transform pipeline.
+/// Currently it checks the following:
/// 1. Region/Block verification: Check the Region/Block verification
/// invariants for every region in the H-CFG.
/// 2. all phi-like recipes must be at the beginning of a block, with no other
/// recipes in between. Note that currently there is still an exception for
/// VPBlendRecipes.
-bool verifyVPlanIsValid(const VPlan &Plan);
+bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate = false);
} // namespace llvm
More information about the llvm-commits
mailing list