[llvm] [VPlan] Add VPSymbolicValueSC for typed VPValues w/o underlying IR value (PR #130507)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 20 12:34:04 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/130507
>From 33813d3abc344b3484edc5acfd15a5532033ad0b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 9 Mar 2025 19:11:59 +0000
Subject: [PATCH 1/3] [VPlan] Add VPSymbolicValueSC for typed VPValues w/o
underlying IR value
This introduces a new VPSymbolicValueSC to be used for typed live-in VPValues
without underlying IR. VPValue is updated to store either the type or
an underlying value in an union. This allows keeping the size of VPValue
unchanged.
The main motivation for adding the type is to not require passing the
canonical IV type to VPTypeAnalysis.
While with this patch, we still need to pass the LLVMContext, this can
also be removed in a future patch (see
9493c38891d7041b86fdcfa1149c7f3ca16773c7)
---
.../Transforms/Vectorize/LoopVectorize.cpp | 10 ++++---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 30 ++++++++++++-------
llvm/lib/Transforms/Vectorize/VPlan.h | 12 ++++----
.../Transforms/Vectorize/VPlanAnalysis.cpp | 9 ++----
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 7 +----
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 4 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 18 ++++++-----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 29 +++++++++++++-----
.../Transforms/Vectorize/VPlanVerifier.cpp | 6 ++--
.../Transforms/Vectorize/VPlanTestBase.h | 5 ++--
11 files changed, 77 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 584da9b7baef5..17fe21e481360 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4478,7 +4478,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
const TargetTransformInfo &TTI) {
assert(VF.isVector() && "Checking a scalar VF?");
- VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+ VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
DenseSet<VPRecipeBase *> EphemeralRecipes;
collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
// Set of already visited types.
@@ -9083,7 +9083,7 @@ static VPInstruction *addResumePhiRecipeForInduction(
/// \p IVEndValues.
static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
DenseMap<VPValue *, VPValue *> &IVEndValues) {
- VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+ VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
auto *ScalarPH = Plan.getScalarPreheader();
auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
@@ -9326,7 +9326,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
return !CM.requiresScalarEpilogue(VF.isVector());
},
Range);
- auto Plan = std::make_unique<VPlan>(OrigLoop);
+ auto Plan =
+ std::make_unique<VPlan>(OrigLoop, Legal->getWidestInductionType());
// Build hierarchical CFG.
// Convert to VPlan-transform and consoliate all transforms for VPlan
// creation.
@@ -9632,7 +9633,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
// Create new empty VPlan
- auto Plan = std::make_unique<VPlan>(OrigLoop);
+ auto Plan =
+ std::make_unique<VPlan>(OrigLoop, Legal->getWidestInductionType());
// Build hierarchical CFG
VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
HCFGBuilder.buildHierarchicalCFG();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index ac8884cfc17ae..68bec6ea49f77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -222,7 +222,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
Loop *CurrentParentLoop, Type *CanonicalIVTy)
: TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
- TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
+ TypeAnalysis(CanonicalIVTy->getContext()), VPDT(*Plan) {}
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
if (Def->isLiveIn())
@@ -850,7 +850,8 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
}
#endif
-VPlan::VPlan(Loop *L) {
+VPlan::VPlan(Loop *L, Type *InductionTy)
+ : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
ScalarHeader = createVPIRBasicBlock(L->getHeader());
@@ -861,7 +862,7 @@ VPlan::VPlan(Loop *L) {
}
VPlan::~VPlan() {
- VPValue DummyValue;
+ VPValue DummyValue((Type *)nullptr);
for (auto *VPB : CreatedBlocks) {
if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB)) {
@@ -891,10 +892,10 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
auto *TCMO = Builder.CreateSub(TripCountV, ConstantInt::get(TCTy, 1),
"trip.count.minus.1");
- BackedgeTakenCount->setUnderlyingValue(TCMO);
+ BackedgeTakenCount->replaceAllUsesWith(getOrAddLiveIn(TCMO));
}
- VectorTripCount.setUnderlyingValue(VectorTripCountV);
+ VectorTripCount.replaceAllUsesWith(getOrAddLiveIn(VectorTripCountV));
IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
// FIXME: Model VF * UF computation completely in VPlan.
@@ -903,12 +904,13 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
unsigned UF = getUF();
if (VF.getNumUsers()) {
Value *RuntimeVF = getRuntimeVF(Builder, TCTy, State.VF);
- VF.setUnderlyingValue(RuntimeVF);
- VFxUF.setUnderlyingValue(
+ VF.replaceAllUsesWith(getOrAddLiveIn(RuntimeVF));
+ VFxUF.replaceAllUsesWith(getOrAddLiveIn(
UF > 1 ? Builder.CreateMul(RuntimeVF, ConstantInt::get(TCTy, UF))
- : RuntimeVF);
+ : RuntimeVF));
} else {
- VFxUF.setUnderlyingValue(createStepForVF(Builder, TCTy, State.VF, UF));
+ VFxUF.replaceAllUsesWith(
+ getOrAddLiveIn(createStepForVF(Builder, TCTy, State.VF, UF)));
}
}
@@ -1175,7 +1177,8 @@ VPlan *VPlan::duplicate() {
return VPIRBB && VPIRBB->getIRBasicBlock() == ScalarHeaderIRBB;
}));
// Create VPlan, clone live-ins and remap operands in the cloned blocks.
- auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
+ auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader,
+ getCanonicalIV()->getScalarType());
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
for (VPValue *OldLiveIn : getLiveIns()) {
Old2NewVPValues[OldLiveIn] =
@@ -1185,7 +1188,7 @@ VPlan *VPlan::duplicate() {
Old2NewVPValues[&VF] = &NewPlan->VF;
Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
if (BackedgeTakenCount) {
- NewPlan->BackedgeTakenCount = new VPValue();
+ NewPlan->BackedgeTakenCount = new VPValue((Type *)nullptr);
Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount;
}
assert(TripCount && "trip count must be set");
@@ -1371,6 +1374,11 @@ static bool isDefinedInsideLoopRegions(const VPValue *VPV) {
DefR->getParent()->getEnclosingLoopRegion());
}
+Type *VPValue::getType() const {
+ assert(isLiveIn());
+ return SubclassID == VPSymbolicValueSC ? Ty : getUnderlyingValue()->getType();
+}
+
bool VPValue::isDefinedOutsideLoopRegions() const {
return !isDefinedInsideLoopRegions(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 3059b87ae63c8..48764c54c640c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3439,8 +3439,9 @@ class VPlan {
/// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
/// wrapping the original header of the scalar loop.
- VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
- : Entry(Entry), ScalarHeader(ScalarHeader) {
+ VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader, Type *InductionTy)
+ : Entry(Entry), ScalarHeader(ScalarHeader), VectorTripCount(InductionTy),
+ VF(InductionTy), VFxUF(InductionTy) {
Entry->setPlan(this);
assert(ScalarHeader->getNumSuccessors() == 0 &&
"scalar header must be a leaf node");
@@ -3450,11 +3451,12 @@ class VPlan {
/// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
/// original preheader and scalar header of \p L, to be used as entry and
/// scalar header blocks of the new VPlan.
- VPlan(Loop *L);
+ VPlan(Loop *L, Type *InductionTy);
/// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
/// wrapping \p ScalarHeaderBB and a trip count of \p TC.
- VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
+ VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC, Type *InductionTy)
+ : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
setEntry(createVPBasicBlock("preheader"));
ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
TripCount = TC;
@@ -3546,7 +3548,7 @@ class VPlan {
/// The backedge taken count of the original loop.
VPValue *getOrCreateBackedgeTakenCount() {
if (!BackedgeTakenCount)
- BackedgeTakenCount = new VPValue();
+ BackedgeTakenCount = new VPValue(getCanonicalIV()->getScalarType());
return BackedgeTakenCount;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 38bec733dbf73..5f05e776dfa22 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -231,13 +231,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
- if (V->isLiveIn()) {
- if (auto *IRValue = V->getLiveInIRValue())
- return IRValue->getType();
- // All VPValues without any underlying IR value (like the vector trip count
- // or the backedge-taken count) have the same type as the canonical IV.
- return CanonicalIVTy;
- }
+ if (V->isLiveIn())
+ return V->getType();
Type *ResultTy =
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index cc21870bee2e3..7d4771d178924 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -39,10 +39,6 @@ class Type;
/// of the previously inferred types.
class VPTypeAnalysis {
DenseMap<const VPValue *, Type *> CachedTypes;
- /// Type of the canonical induction variable. Used for all VPValues without
- /// any underlying IR value (like the vector trip count or the backedge-taken
- /// count).
- Type *CanonicalIVTy;
LLVMContext &Ctx;
Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
@@ -55,8 +51,7 @@ class VPTypeAnalysis {
Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
public:
- VPTypeAnalysis(Type *CanonicalIVTy)
- : CanonicalIVTy(CanonicalIVTy), Ctx(CanonicalIVTy->getContext()) {}
+ VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {}
/// Infer the type of \p V. Returns the scalar type of \p V.
Type *inferScalarType(const VPValue *V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index d53b72bb2258a..1ba55d6ac15ee 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -379,8 +379,8 @@ struct VPCostContext {
VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
Type *CanIVTy, LoopVectorizationCostModel &CM,
TargetTransformInfo::TargetCostKind CostKind)
- : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
- CM(CM), CostKind(CostKind) {}
+ : TTI(TTI), TLI(TLI), Types(CanIVTy->getContext()),
+ LLVMCtx(CanIVTy->getContext()), CM(CM), CostKind(CostKind) {}
/// Return the cost for \p UI with \p VF using the legacy cost model as
/// fallback until computing the cost of all recipes migrates to VPlan.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6c8680ac96610..2de6f3b83c1d2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -549,7 +549,7 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
// Truncate base induction if needed.
Type *CanonicalIVType = CanonicalIV->getScalarType();
- VPTypeAnalysis TypeInfo(CanonicalIVType);
+ VPTypeAnalysis TypeInfo(CanonicalIVType->getContext());
Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
if (TruncI) {
Type *TruncTy = TruncI->getType();
@@ -795,13 +795,14 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
void VPlanTransforms::optimizeInductionExitUsers(
VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
- VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+ VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
for (VPRecipeBase &R : *ExitVPBB) {
auto *ExitIRI = cast<VPIRInstruction>(&R);
if (!isa<PHINode>(ExitIRI->getInstruction()))
break;
+
for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
if (PredVPBB == MiddleVPBB)
if (VPValue *Escape = optimizeLatchExitInductionUser(
@@ -957,8 +958,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
#ifndef NDEBUG
// Verify that the cached type info is for both A and its users is still
// accurate by comparing it to freshly computed types.
- VPTypeAnalysis TypeInfo2(
- R.getParent()->getPlan()->getCanonicalIV()->getScalarType());
+ VPTypeAnalysis TypeInfo2(R.getParent()
+ ->getPlan()
+ ->getCanonicalIV()
+ ->getScalarType()
+ ->getContext());
assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A));
for (VPUser *U : A->users()) {
auto *R = cast<VPRecipeBase>(U);
@@ -1001,7 +1005,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
- VPTypeAnalysis TypeInfo(&CanonicalIVTy);
+ VPTypeAnalysis TypeInfo(CanonicalIVTy.getContext());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
simplifyRecipe(R, TypeInfo);
@@ -1351,7 +1355,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
// typed.
DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
- VPTypeAnalysis TypeInfo(CanonicalIVType);
+ VPTypeAnalysis TypeInfo(CanonicalIVType->getContext());
VPBasicBlock *PH = Plan.getVectorPreheader();
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
@@ -1743,8 +1747,8 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
/// Replace recipes with their EVL variants.
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
- VPTypeAnalysis TypeInfo(CanonicalIVType);
LLVMContext &Ctx = CanonicalIVType->getContext();
+ VPTypeAnalysis TypeInfo(Ctx);
VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index a36c2aeb3da5c..0e33bd77bc30d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -73,7 +73,7 @@ class UnrollState {
public:
UnrollState(VPlan &Plan, unsigned UF, LLVMContext &Ctx)
- : Plan(Plan), UF(UF), TypeInfo(Plan.getCanonicalIV()->getScalarType()) {}
+ : Plan(Plan), UF(UF), TypeInfo(Ctx) {}
void unrollBlock(VPBlockBase *VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index d0fa62978c1b4..fdad00cb03313 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -61,8 +61,12 @@ class VPValue {
SmallVector<VPUser *, 1> Users;
protected:
- // Hold the underlying Value, if any, attached to this VPValue.
- Value *UnderlyingVal;
+ union {
+ // Hold the underlying Value, if any, attached to this non-symbolic VPValue.
+ Value *UnderlyingVal;
+ // Hold the type of this VPValue, if it is symbolic.
+ Type *Ty;
+ };
/// Pointer to the VPDef that defines this VPValue. If it is nullptr, the
/// VPValue is not defined by any recipe modeled in VPlan.
@@ -70,8 +74,9 @@ class VPValue {
VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
- /// Create a live-in VPValue.
- VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {}
+ /// Create a live-in IR VPValue.
+ VPValue(Value *UV) : VPValue(VPValueSC, UV, nullptr) {}
+ VPValue(Type *Ty) : SubclassID(VPSymbolicValueSC), Ty(Ty), Def(nullptr) {}
/// Create a VPValue for a \p Def which is a subclass of VPValue.
VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {}
/// Create a VPValue for a \p Def which defines multiple values.
@@ -86,14 +91,18 @@ class VPValue {
public:
/// Return the underlying Value attached to this VPValue.
- Value *getUnderlyingValue() const { return UnderlyingVal; }
+ Value *getUnderlyingValue() const {
+ return SubclassID == VPSymbolicValueSC ? nullptr : UnderlyingVal;
+ }
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe
- /// that defines multiple values.
- VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase.
+ VPValueSC, /// A generic non-symbolic VPValue, like live-in IR values or
+ /// defined by a recipe that defines multiple values.
+ VPSymbolicValueSC, /// A generic VPValue, like live-in values or defined by
+ /// a recipe that defines multiple values.
+ VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase.
};
VPValue(const VPValue &) = delete;
@@ -172,6 +181,10 @@ class VPValue {
/// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
bool isLiveIn() const { return !hasDefiningRecipe(); }
+ bool isSymbolic() const { return SubclassID == VPSymbolicValueSC; }
+
+ Type *getType() const;
+
/// Returns the underlying IR value, if this VPValue is defined outside the
/// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef
/// inside a VPlan.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 6fe131879b1a2..16f1e70b4e753 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -420,8 +420,10 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
bool llvm::verifyVPlanIsValid(const VPlan &Plan) {
VPDominatorTree VPDT;
VPDT.recalculate(const_cast<VPlan &>(Plan));
- VPTypeAnalysis TypeInfo(
- const_cast<VPlan &>(Plan).getCanonicalIV()->getScalarType());
+ VPTypeAnalysis TypeInfo(const_cast<VPlan &>(Plan)
+ .getCanonicalIV()
+ ->getScalarType()
+ ->getContext());
VPlanVerifier Verifier(VPDT, TypeInfo);
return Verifier.verify(Plan);
}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index caf5d2357411d..85ac0c3ac8a75 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -71,7 +71,7 @@ class VPlanTestIRBase : public testing::Test {
Loop *L = LI->getLoopFor(LoopHeader);
PredicatedScalarEvolution PSE(*SE, *L);
- auto Plan = std::make_unique<VPlan>(L);
+ auto Plan = std::make_unique<VPlan>(L, IntegerType::get(*Ctx, 64));
VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan);
HCFGBuilder.buildHierarchicalCFG();
VPlanTransforms::introduceTopLevelVectorLoopRegion(
@@ -91,7 +91,8 @@ class VPlanTestBase : public testing::Test {
}
VPlan &getPlan(VPValue *TC = nullptr) {
- Plans.push_back(std::make_unique<VPlan>(&*ScalarHeader, TC));
+ Plans.push_back(
+ std::make_unique<VPlan>(&*ScalarHeader, TC, IntegerType::get(C, 64)));
return *Plans.back();
}
};
>From 7f9235c47369299b47045ff0809666e7c54dc41d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 20 Mar 2025 19:17:23 +0000
Subject: [PATCH 2/3] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 6 ++++--
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 68bec6ea49f77..7ad1340cdea60 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1375,7 +1375,7 @@ static bool isDefinedInsideLoopRegions(const VPValue *VPV) {
}
Type *VPValue::getType() const {
- assert(isLiveIn());
+ assert(isLiveIn() && "can only return the type for a live-in");
return SubclassID == VPSymbolicValueSC ? Ty : getUnderlyingValue()->getType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 48764c54c640c..54614405d3832 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3454,7 +3454,7 @@ class VPlan {
VPlan(Loop *L, Type *InductionTy);
/// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
- /// wrapping \p ScalarHeaderBB and a trip count of \p TC.
+ /// wrapping \p ScalarHeaderBB and a trip count of \p TC. Also creates symbolic VectorTripCount, VF and VFxUF VPValues using \p InductionTy.
VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC, Type *InductionTy)
: VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
setEntry(createVPBasicBlock("preheader"));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index fdad00cb03313..0f6de92b2be2e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -92,7 +92,7 @@ class VPValue {
public:
/// Return the underlying Value attached to this VPValue.
Value *getUnderlyingValue() const {
- return SubclassID == VPSymbolicValueSC ? nullptr : UnderlyingVal;
+ return isSymbolic() ? nullptr : UnderlyingVal;
}
/// An enumeration for keeping track of the concrete subclass of VPValue that
@@ -181,9 +181,11 @@ class VPValue {
/// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
bool isLiveIn() const { return !hasDefiningRecipe(); }
+ /// Returns true if the VPValue is symbolic, that is a live-in without underlying value.
bool isSymbolic() const { return SubclassID == VPSymbolicValueSC; }
- Type *getType() const;
+ /// If the VPValue is a live-in, return its scalar type.
+ Type *getScalarType() const;
/// Returns the underlying IR value, if this VPValue is defined outside the
/// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef
>From acbca4ae4a710429d21c9be1a304f3e70aa2d004 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 20 Mar 2025 19:33:35 +0000
Subject: [PATCH 3/3] !fixup fix formatting
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 54614405d3832..d81afa1bf406e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3454,7 +3454,8 @@ class VPlan {
VPlan(Loop *L, Type *InductionTy);
/// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
- /// wrapping \p ScalarHeaderBB and a trip count of \p TC. Also creates symbolic VectorTripCount, VF and VFxUF VPValues using \p InductionTy.
+ /// wrapping \p ScalarHeaderBB and a trip count of \p TC. Also creates
+ /// symbolic VectorTripCount, VF and VFxUF VPValues using \p InductionTy.
VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC, Type *InductionTy)
: VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
setEntry(createVPBasicBlock("preheader"));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 0f6de92b2be2e..c1f95aa24d5e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -181,7 +181,8 @@ class VPValue {
/// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
bool isLiveIn() const { return !hasDefiningRecipe(); }
- /// Returns true if the VPValue is symbolic, that is a live-in without underlying value.
+ /// Returns true if the VPValue is symbolic, that is a live-in without
+ /// underlying value.
bool isSymbolic() const { return SubclassID == VPSymbolicValueSC; }
/// If the VPValue is a live-in, return its scalar type.
More information about the llvm-commits
mailing list