[llvm] [VPlan] Add helper to run VPlan passes, verify after run (NFC). (PR #123640)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 28 06:33:53 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/123640
>From 5d66f2be2398c6f9499f8928dd68bc5f83e753ed Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 20 Jan 2025 14:48:17 +0000
Subject: [PATCH 1/3] [VPlan] Add helper to run VPlan passes, verify after run
(NFC).
Add new runPass helpers to run a VPlan transformation. This makes it
easier to add additional checks/functionality for each transform run. In
this patch, an option is added to run the verifier after each VPlan
transform.
Follow-ups will use the same helper to also support printing VPlans
after each transform.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 45 ++++++++++++-------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 1 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 24 +++++-----
.../Transforms/Vectorize/VPlanTransforms.h | 29 +++++++++++-
4 files changed, 71 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 57b7358049bcef..1ed05145855ac6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -355,6 +355,16 @@ cl::opt<bool> EnableVPlanNativePath(
"enable-vplan-native-path", cl::Hidden,
cl::desc("Enable VPlan-native vectorization path with "
"support for outer loop vectorization."));
+
+cl::opt<bool>
+ VerifyEachVPlan("vplan-verify-each",
+#ifdef EXPENSIVE_CHECKS
+ cl::init(true),
+#else
+ cl::init(false),
+#endif
+ cl::Hidden,
+ cl::desc("Verfiy VPlans after VPlan transforms."));
} // namespace llvm
// This flag enables the stress testing of the VPlan H-CFG construction in the
@@ -7651,8 +7661,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
- VPlanTransforms::unrollByUF(BestVPlan, BestUF,
- OrigLoop->getHeader()->getContext());
+ VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
+ OrigLoop->getHeader()->getContext());
VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
VPlanTransforms::convertToConcreteRecipes(BestVPlan);
@@ -8908,13 +8918,14 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
// Now optimize the initial VPlan.
if (!Plan->hasVF(ElementCount::getFixed(1)))
- VPlanTransforms::truncateToMinimalBitwidths(*Plan,
- CM.getMinimalBitwidths());
+ VPlanTransforms::runPass(VPlanTransforms::truncateToMinimalBitwidths,
+ *Plan, CM.getMinimalBitwidths());
VPlanTransforms::optimize(*Plan);
// TODO: try to put it close to addActiveLaneMask().
// Discard the plan if it is not EVL-compatible
- if (CM.foldTailWithEVL() && !VPlanTransforms::tryAddExplicitVectorLength(
- *Plan, CM.getMaxSafeElements()))
+ if (CM.foldTailWithEVL() &&
+ !VPlanTransforms::runPass(VPlanTransforms::tryAddExplicitVectorLength,
+ *Plan, CM.getMaxSafeElements()))
break;
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
VPlans.push_back(std::move(Plan));
@@ -9396,7 +9407,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
if (auto *UncountableExitingBlock =
Legal->getUncountableEarlyExitingBlock()) {
- if (!VPlanTransforms::handleUncountableEarlyExit(
+ if (!VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit,
*Plan, *PSE.getSE(), OrigLoop, UncountableExitingBlock,
RecipeBuilder)) {
reportVectorizationFailure(
@@ -9423,8 +9434,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
// Interleave memory: for each Interleave Group we marked earlier as relevant
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
- VPlanTransforms::createInterleaveGroups(
- *Plan, InterleaveGroups, RecipeBuilder, CM.isScalarEpilogueAllowed());
+ VPlanTransforms::runPass(VPlanTransforms::createInterleaveGroups, *Plan,
+ InterleaveGroups, RecipeBuilder,
+ CM.isScalarEpilogueAllowed());
for (ElementCount VF : Range)
Plan->addVF(VF);
@@ -9466,13 +9478,16 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
}
}
- VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) {
+ auto BlockNeedsPredication = [this](BasicBlock *BB) {
return Legal->blockNeedsPredication(BB);
- });
+ };
+ VPlanTransforms::runPass(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan,
+ BlockNeedsPredication);
// Sink users of fixed-order recurrence past the recipe defining the previous
// value and introduce FirstOrderRecurrenceSplice VPInstructions.
- if (!VPlanTransforms::adjustFixedOrderRecurrences(*Plan, Builder))
+ if (!VPlanTransforms::runPass(VPlanTransforms::adjustFixedOrderRecurrences,
+ *Plan, Builder))
return nullptr;
if (useActiveLaneMask(Style)) {
@@ -9815,10 +9830,10 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
}
}
-
- VPlanTransforms::clearReductionWrapFlags(*Plan);
for (VPRecipeBase *R : ToDelete)
R->eraseFromParent();
+
+ VPlanTransforms::runPass(VPlanTransforms::clearReductionWrapFlags, *Plan);
}
void VPDerivedIVRecipe::execute(VPTransformState &State) {
@@ -10182,7 +10197,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
VPIRInst->eraseFromParent();
ResumePhi->eraseFromParent();
}
- VPlanTransforms::removeDeadRecipes(MainPlan);
+ VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan);
using namespace VPlanPatternMatch;
VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 35da93ee3b407c..52d15c535276af 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -52,6 +52,7 @@ using namespace llvm::VPlanPatternMatch;
namespace llvm {
extern cl::opt<bool> EnableVPlanNativePath;
}
+
extern cl::opt<unsigned> ForceTargetInstructionCost;
static cl::opt<bool> PrintVPlansInDotFormat(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 714250a56ff576..e147ccbd15d1b4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -19,6 +19,7 @@
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
+#include "VPlanVerifier.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
@@ -1439,19 +1440,19 @@ void VPlanTransforms::truncateToMinimalBitwidths(
}
void VPlanTransforms::optimize(VPlan &Plan) {
- removeRedundantCanonicalIVs(Plan);
- removeRedundantInductionCasts(Plan);
+ runPass(removeRedundantCanonicalIVs, Plan);
+ runPass(removeRedundantInductionCasts, Plan);
simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType());
- removeDeadRecipes(Plan);
- legalizeAndOptimizeInductions(Plan);
- removeRedundantExpandSCEVRecipes(Plan);
+ runPass(removeDeadRecipes, Plan);
+ runPass(legalizeAndOptimizeInductions, Plan);
+ runPass(removeRedundantExpandSCEVRecipes, Plan);
simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType());
- removeDeadRecipes(Plan);
+ runPass(removeDeadRecipes, Plan);
- createAndOptimizeReplicateRegions(Plan);
- mergeBlocksIntoPredecessors(Plan);
- licm(Plan);
+ runPass(createAndOptimizeReplicateRegions, Plan);
+ runPass(mergeBlocksIntoPredecessors, Plan);
+ runPass(licm, Plan);
}
// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
@@ -1871,7 +1872,8 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
}
void VPlanTransforms::dropPoisonGeneratingRecipes(
- VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication) {
+ VPlan &Plan,
+ const std::function<bool(BasicBlock *)> &BlockNeedsPredication) {
// Collect recipes in the backward slice of `Root` that may generate a poison
// value that is used after vectorization.
SmallPtrSet<VPRecipeBase *, 16> Visited;
@@ -1971,7 +1973,7 @@ void VPlanTransforms::createInterleaveGroups(
VPlan &Plan,
const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
&InterleaveGroups,
- VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed) {
+ VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed) {
if (InterleaveGroups.empty())
return;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b31fef5d62456b..ad24d9f1468222 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -14,7 +14,9 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
#include "VPlan.h"
+#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/Support/CommandLine.h"
namespace llvm {
@@ -27,7 +29,29 @@ class TargetLibraryInfo;
class VPBuilder;
class VPRecipeBuilder;
+extern cl::opt<bool> VerifyEachVPlan;
+
struct VPlanTransforms {
+ /// Helper to run a VPlan transform \p Transform on \p VPlan, forwarding extra
+ /// arguments to the transform. Returns the boolean returned by the transform.
+ template <typename... ArgsTy>
+ static bool runPass(bool (*Transform)(VPlan &, ArgsTy...), VPlan &Plan,
+ typename std::remove_reference<ArgsTy>::type &...Args) {
+ bool Res = Transform(Plan, Args...);
+ if (VerifyEachVPlan)
+ verifyVPlanIsValid(Plan);
+ return Res;
+ }
+ /// Helper to run a VPlan transform \p Transform on \p VPlan, forwarding extra
+ /// arguments to the transform.
+ template <typename... ArgsTy>
+ static void runPass(void (*Fn)(VPlan &, ArgsTy...), VPlan &Plan,
+ typename std::remove_reference<ArgsTy>::type &...Args) {
+ Fn(Plan, Args...);
+ if (VerifyEachVPlan)
+ verifyVPlanIsValid(Plan);
+ }
+
/// Replaces the VPInstructions in \p Plan with corresponding
/// widen recipes.
static void
@@ -100,7 +124,8 @@ struct VPlanTransforms {
/// TODO: Replace BlockNeedsPredication callback with retrieving info from
/// VPlan directly.
static void dropPoisonGeneratingRecipes(
- VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication);
+ VPlan &Plan,
+ const std::function<bool(BasicBlock *)> &BlockNeedsPredication);
/// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
/// replaces all uses except the canonical IV increment of
@@ -119,7 +144,7 @@ struct VPlanTransforms {
VPlan &Plan,
const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
&InterleaveGroups,
- VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed);
+ VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed);
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
>From b63b50e3f46808b4cbb9ea1b9fa22d9d8b2df13c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 28 Jan 2025 14:07:24 +0000
Subject: [PATCH 2/3] !fixup also use runPass for simplifyRecipes.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e147ccbd15d1b4..242dc8a636a6f8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -965,10 +965,10 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
/// Try to simplify the recipes in \p Plan. Use \p CanonicalIVTy as type for all
/// un-typed live-ins in VPTypeAnalysis.
-static void simplifyRecipes(VPlan &Plan, Type *CanonicalIVTy) {
+static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
- VPTypeAnalysis TypeInfo(CanonicalIVTy);
+ VPTypeAnalysis TypeInfo(&CanonicalIVTy);
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
simplifyRecipe(R, TypeInfo);
@@ -1030,7 +1030,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
VPBlockUtils::connectBlocks(Preheader, Header);
VPBlockUtils::connectBlocks(ExitingVPBB, Exit);
- simplifyRecipes(Plan, CanIVTy);
+ simplifyRecipes(Plan, *CanIVTy);
} else {
// The vector region contains header phis for which we cannot remove the
// loop region yet.
@@ -1443,11 +1443,11 @@ void VPlanTransforms::optimize(VPlan &Plan) {
runPass(removeRedundantCanonicalIVs, Plan);
runPass(removeRedundantInductionCasts, Plan);
- simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType());
+ runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
runPass(removeDeadRecipes, Plan);
runPass(legalizeAndOptimizeInductions, Plan);
runPass(removeRedundantExpandSCEVRecipes, Plan);
- simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType());
+ runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
runPass(removeDeadRecipes, Plan);
runPass(createAndOptimizeReplicateRegions, Plan);
>From 5c1ca672bddf4d90908f6acb4e6339d547689022 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 28 Jan 2025 14:24:40 +0000
Subject: [PATCH 3/3] !fixup opt out handleUncountableEarlyExit for now
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1ed05145855ac6..5375d2be9c8751 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9407,7 +9407,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
if (auto *UncountableExitingBlock =
Legal->getUncountableEarlyExitingBlock()) {
- if (!VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit,
+ if (!VPlanTransforms::handleUncountableEarlyExit(
*Plan, *PSE.getSE(), OrigLoop, UncountableExitingBlock,
RecipeBuilder)) {
reportVectorizationFailure(
More information about the llvm-commits
mailing list