[llvm] [VPlan] Remove Def pointer from VPRecipeValue for single-def recipes (PR #195483)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat May 2 14:04:17 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/195483
For VPSingleDefRecipe, the VPRecipeValue's Def pointer always points back to the containing VPRecipeBase, which is computable via static_cast.
Introduce 2 VPRecipeValue subclasses to distinguish the VPValues defined by VPSingleDefRecipes (VPSingleDefValue), and VPStandaloneValue for other recipes.
The former does not need to store a pointer to the defining recipe, as it can be computed via static_cast. This saves 8 bytes for most recipes. I plan to use the extra bytes to store the type directly in VPValue as follow-ups.
>From 040de2bbf319055fd138351d5e1ca34f36819e47 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 11 Apr 2026 17:19:53 +0100
Subject: [PATCH] [VPlan] Remove Def pointer from VPRecipeValue for single-def
recipes
For VPSingleDefRecipe, the VPRecipeValue's Def pointer always points
back to the containing VPRecipeBase, which is computable via
static_cast.
Introduce 2 VPRecipeValue subclasses to distinguish the VPValues defined
by VPSingleDefRecipes (VPSingleDefValue), and VPStandaloneValue for
other recipes.
The former does not need to store a pointer to the defining recipe, as
it can be computed via static_cast. This saves 8 bytes for most recipes.
I plan to use the extra bytes to store the type directly in VPValue as
follow-ups.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 31 +++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 22 +++---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 70 ++++++++++++++-----
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
4 files changed, 89 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 91fa4c57e5bf9..cf7ebf8c3d2a8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -122,17 +122,22 @@ void VPRecipeBase::dump() const {
#endif
#if !defined(NDEBUG)
-bool VPRecipeValue::isDefinedBy(const VPDef *D) const { return Def == D; }
+bool VPRecipeValue::isDefinedBy(const VPDef *D) const {
+ return getDefiningRecipe() == D;
+}
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ if (!DefValue)
+ return nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(DefValue))
+ return SV->getDef();
+ return static_cast<VPSingleDefRecipe *>(DefValue);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ return const_cast<VPValue *>(this)->getDefiningRecipe();
}
Value *VPValue::getLiveInIRValue() const {
@@ -141,15 +146,21 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPRecipeValue::VPRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPValue(VPVRecipeValueSC, UV), Def(Def) {
- assert(Def && "VPRecipeValue requires a defining recipe");
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
+ : VPRecipeValue(VPVSingleDefValueSC, UV) {
+ assert(Def && "VPSingleDefValue requires a defining recipe");
+ Def->addDefinedValue(this);
+}
+
+VPRecipeValue::~VPRecipeValue() = default;
+
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+ assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPRecipeValue::~VPRecipeValue() {
- assert(Users.empty() &&
- "trying to delete a VPRecipeValue with remaining users");
+VPStandaloneRecipeValue::~VPStandaloneRecipeValue() {
Def->removeDefinedValue(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 592d83af2295f..783d3a1f8d17d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -601,16 +601,17 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
-/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+
+ ~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -2869,7 +2870,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst);
}
} else {
for (auto *SV : StoredValues)
@@ -3576,13 +3577,14 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
+ : public VPWidenMemoryRecipe,
+ public VPStandaloneRecipeValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecutive, Metadata, DL),
- VPRecipeValue(this, &Load) {
+ VPStandaloneRecipeValue(this, &Load) {
setMask(Mask);
}
@@ -3617,13 +3619,13 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// using the address to load from, the explicit vector length and an optional
/// mask.
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+ public VPStandaloneRecipeValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L,
L.getDebugLoc()),
- VPRecipeValue(this, &getIngredient()) {
+ VPStandaloneRecipeValue(this, &getIngredient()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8f9806adf774e..4cb911b8e74c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -39,6 +39,7 @@ class VPDef;
class VPSlotTracker;
class VPUser;
class VPRecipeBase;
+class VPSingleDefRecipe;
class VPPhiAccessors;
class VPRegionValue;
class VPRegionBlock;
@@ -79,11 +80,12 @@ class LLVM_ABI_FOR_TEST VPValue {
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
- VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
- VPVRecipeValueSC, /// A VPValue defined by a recipe.
- VPRegionValueSC, /// A VPValue sub-class that is defined by a region, like
- /// the canonical IV of a loop region.
+ VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
+ VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
+ VPVStandaloneRecipeValueSC, /// A standalone VPValue defined by a recipe.
+ VPVSingleDefValueSC, /// A VPValue embedded in a VPSingleDefRecipe.
+ VPRegionValueSC, /// A VPValue sub-class that is defined by a
+ /// region, like a loop region canonical IV.
};
VPValue(const VPValue &) = delete;
@@ -299,14 +301,11 @@ struct VPSymbolicValue : public VPValue {
bool Materialized = false;
};
-/// A VPValue defined by a recipe that produces one or more values.
+/// Abstract base class for VPValues defined by a VPRecipeBase.
class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
- /// Pointer to the VPRecipeBase that defines this VPValue.
- VPRecipeBase *Def;
-
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
/// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
@@ -314,13 +313,51 @@ class VPRecipeValue : public VPValue {
bool isDefinedBy(const VPDef *D) const;
#endif
+protected:
+ VPRecipeValue(unsigned char SC, Value *UV = nullptr) : VPValue(SC, UV) {}
+
+public:
+ LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
+ V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue embedded as a subobject of VPSingleDefRecipe.
+class VPSingleDefValue : public VPRecipeValue {
+ friend class VPDef;
+ friend class VPSingleDefRecipe;
+
+protected:
+ /// Construct a VPSingleDefValue. Must only be used by VPSingleDefRecipe.
+ LLVM_ABI_FOR_TEST VPSingleDefValue(VPSingleDefRecipe *Def,
+ Value *UV = nullptr);
+
+public:
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue that stores a pointer to its defining recipe.
+class VPStandaloneRecipeValue : public VPRecipeValue {
+ friend class VPDef;
+
+ /// Pointer to the VPRecipeBase that defines this VPValue.
+ VPRecipeBase *Def;
+
public:
- LLVM_ABI_FOR_TEST VPRecipeValue(VPRecipeBase *Def, Value *UV = nullptr);
+ LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
+ Value *UV = nullptr);
- LLVM_ABI_FOR_TEST virtual ~VPRecipeValue();
+ ~VPStandaloneRecipeValue() override;
+
+ VPRecipeBase *getDef() const { return Def; }
static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVRecipeValueSC;
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC;
}
};
@@ -430,15 +467,15 @@ class VPUser {
/// Single-value VPDefs that also inherit from VPValue must make sure to inherit
/// from VPDef before VPValue.
class VPDef {
- friend class VPRecipeValue;
+ friend class VPSingleDefValue;
+ friend class VPSingleDefRecipe;
+ friend class VPStandaloneRecipeValue;
/// The VPValues defined by this VPDef.
TinyPtrVector<VPRecipeValue *> DefinedValues;
/// Add \p V as a defined value by this VPDef.
void addDefinedValue(VPRecipeValue *V) {
- assert(V->isDefinedBy(this) &&
- "can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
}
@@ -450,7 +487,8 @@ class VPDef {
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
llvm::erase(DefinedValues, V);
- V->Def = nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(V))
+ SV->Def = nullptr;
}
public:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a1ddda7eda969..9cc1d90d6256d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1698,8 +1698,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPRecipeValue(this);
- new VPRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
More information about the llvm-commits
mailing list