[llvm] [VPlan] Remove Def pointer from for VPSingleDefRecipes (NFC) (PR #195483)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon May 18 02:03:22 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/195483
>From 2ffc309fc77529f4d7d27c7003af97c98352d54c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 11 Apr 2026 17:19:53 +0100
Subject: [PATCH 1/6] [VPlan] Remove Def pointer from VPRecipeValue for
single-def recipes
For VPSingleDefRecipe, the VPRecipeValue's Def pointer always points
back to the containing VPRecipeBase, which is computable via
static_cast.
Introduce 2 VPRecipeValue subclasses to distinguish the VPValues defined
by VPSingleDefRecipes (VPSingleDefValue), and VPStandaloneValue for
other recipes.
The former does not need to store a pointer to the defining recipe, as
it can be computed via static_cast. This saves 8 bytes for most recipes.
I plan to use the extra bytes to store the type directly in VPValue as
follow-ups.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 31 +++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 22 +++---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 70 ++++++++++++++-----
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
4 files changed, 89 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2ec1b002f56f8..0c13e2d1c93eb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -122,17 +122,22 @@ void VPRecipeBase::dump() const {
#endif
#if !defined(NDEBUG)
-bool VPRecipeValue::isDefinedBy(const VPDef *D) const { return Def == D; }
+bool VPRecipeValue::isDefinedBy(const VPDef *D) const {
+ return getDefiningRecipe() == D;
+}
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ if (!DefValue)
+ return nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(DefValue))
+ return SV->getDef();
+ return static_cast<VPSingleDefRecipe *>(DefValue);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ return const_cast<VPValue *>(this)->getDefiningRecipe();
}
Value *VPValue::getLiveInIRValue() const {
@@ -141,15 +146,21 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPRecipeValue::VPRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPValue(VPVRecipeValueSC, UV), Def(Def) {
- assert(Def && "VPRecipeValue requires a defining recipe");
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
+ : VPRecipeValue(VPVSingleDefValueSC, UV) {
+ assert(Def && "VPSingleDefValue requires a defining recipe");
+ Def->addDefinedValue(this);
+}
+
+VPRecipeValue::~VPRecipeValue() = default;
+
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+ assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPRecipeValue::~VPRecipeValue() {
- assert(Users.empty() &&
- "trying to delete a VPRecipeValue with remaining users");
+VPStandaloneRecipeValue::~VPStandaloneRecipeValue() {
Def->removeDefinedValue(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f6e77092e016c..3cc088eb28446 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -605,16 +605,17 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
-/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+
+ ~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -2873,7 +2874,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst);
}
} else {
for (auto *SV : StoredValues)
@@ -3568,13 +3569,14 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
+ : public VPWidenMemoryRecipe,
+ public VPStandaloneRecipeValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecutive, Metadata, DL),
- VPRecipeValue(this, &Load) {
+ VPStandaloneRecipeValue(this, &Load) {
setMask(Mask);
}
@@ -3609,13 +3611,13 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// using the address to load from, the explicit vector length and an optional
/// mask.
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+ public VPStandaloneRecipeValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L,
L.getDebugLoc()),
- VPRecipeValue(this, &getIngredient()) {
+ VPStandaloneRecipeValue(this, &getIngredient()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 777da17c904f3..1af914b528e77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -39,6 +39,7 @@ class VPDef;
class VPSlotTracker;
class VPUser;
class VPRecipeBase;
+class VPSingleDefRecipe;
class VPPhiAccessors;
class VPRegionValue;
class VPRegionBlock;
@@ -79,11 +80,12 @@ class LLVM_ABI_FOR_TEST VPValue {
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
- VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
- VPVRecipeValueSC, /// A VPValue defined by a recipe.
- VPRegionValueSC, /// A VPValue sub-class that is defined by a region, like
- /// the canonical IV of a loop region.
+ VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
+ VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
+ VPVStandaloneRecipeValueSC, /// A standalone VPValue defined by a recipe.
+ VPVSingleDefValueSC, /// A VPValue embedded in a VPSingleDefRecipe.
+ VPRegionValueSC, /// A VPValue sub-class that is defined by a
+ /// region, like a loop region canonical IV.
};
VPValue(const VPValue &) = delete;
@@ -305,14 +307,11 @@ struct VPSymbolicValue : public VPValue {
bool Materialized = false;
};
-/// A VPValue defined by a recipe that produces one or more values.
+/// Abstract base class for VPValues defined by a VPRecipeBase.
class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
- /// Pointer to the VPRecipeBase that defines this VPValue.
- VPRecipeBase *Def;
-
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
/// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
@@ -320,13 +319,51 @@ class VPRecipeValue : public VPValue {
bool isDefinedBy(const VPDef *D) const;
#endif
+protected:
+ VPRecipeValue(unsigned char SC, Value *UV = nullptr) : VPValue(SC, UV) {}
+
+public:
+ LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
+ V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue embedded as a subobject of VPSingleDefRecipe.
+class VPSingleDefValue : public VPRecipeValue {
+ friend class VPDef;
+ friend class VPSingleDefRecipe;
+
+protected:
+ /// Construct a VPSingleDefValue. Must only be used by VPSingleDefRecipe.
+ LLVM_ABI_FOR_TEST VPSingleDefValue(VPSingleDefRecipe *Def,
+ Value *UV = nullptr);
+
+public:
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue that stores a pointer to its defining recipe.
+class VPStandaloneRecipeValue : public VPRecipeValue {
+ friend class VPDef;
+
+ /// Pointer to the VPRecipeBase that defines this VPValue.
+ VPRecipeBase *Def;
+
public:
- LLVM_ABI_FOR_TEST VPRecipeValue(VPRecipeBase *Def, Value *UV = nullptr);
+ LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
+ Value *UV = nullptr);
- LLVM_ABI_FOR_TEST virtual ~VPRecipeValue();
+ ~VPStandaloneRecipeValue() override;
+
+ VPRecipeBase *getDef() const { return Def; }
static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVRecipeValueSC;
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC;
}
};
@@ -436,15 +473,15 @@ class VPUser {
/// Single-value VPDefs that also inherit from VPValue must make sure to inherit
/// from VPDef before VPValue.
class VPDef {
- friend class VPRecipeValue;
+ friend class VPSingleDefValue;
+ friend class VPSingleDefRecipe;
+ friend class VPStandaloneRecipeValue;
/// The VPValues defined by this VPDef.
TinyPtrVector<VPRecipeValue *> DefinedValues;
/// Add \p V as a defined value by this VPDef.
void addDefinedValue(VPRecipeValue *V) {
- assert(V->isDefinedBy(this) &&
- "can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
}
@@ -456,7 +493,8 @@ class VPDef {
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
llvm::erase(DefinedValues, V);
- V->Def = nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(V))
+ SV->Def = nullptr;
}
public:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 1dceed39b2d03..7cd7077b6116a 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1736,8 +1736,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPRecipeValue(this);
- new VPRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
>From c43a3b457d9c5c2491357c36fb4a726730de0b8f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 12 May 2026 21:25:04 +0100
Subject: [PATCH 2/6] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 5 +++--
llvm/lib/Transforms/Vectorize/VPlanValue.h | 10 ++++++----
3 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 0c13e2d1c93eb..23c71f9e5da1a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -146,14 +146,14 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
+VPRecipeValue::~VPRecipeValue() = default;
+
VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
: VPRecipeValue(VPVSingleDefValueSC, UV) {
assert(Def && "VPSingleDefValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPRecipeValue::~VPRecipeValue() = default;
-
VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
: VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 3cc088eb28446..1899513a6453a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -603,8 +603,9 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
return R->getVPRecipeID() == VPRecipeID; \
}
-/// VPSingleDef is a base class for recipes for modeling a sequence of one or
-/// more output IR that define a single result VPValue.
+/// VPSingleDefRecipe is a base class for recipes for modeling a sequence of one
+/// or more output IR that define a single result VPValue. Note that
+/// VPRecipeBase must be inherited from before VPValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 1af914b528e77..849dfa2649da9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -39,10 +39,10 @@ class VPDef;
class VPSlotTracker;
class VPUser;
class VPRecipeBase;
-class VPSingleDefRecipe;
class VPPhiAccessors;
class VPRegionValue;
class VPRegionBlock;
+class VPSingleDefRecipe;
/// This is the base class of the VPlan Def/Use graph, used for modeling the
/// data flow into, within and out of the VPlan. VPValues can stand for live-ins
@@ -314,8 +314,8 @@ class VPRecipeValue : public VPValue {
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
- /// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
- /// /p D are associated with its VPRecipeBase,
+ /// NOTE: Only used by VPDef to assert that VPRecipeValues removed from
+ /// /p D are associated with its VPRecipeBase.
bool isDefinedBy(const VPDef *D) const;
#endif
@@ -331,7 +331,7 @@ class VPRecipeValue : public VPValue {
}
};
-/// A VPRecipeValue embedded as a subobject of VPSingleDefRecipe.
+/// A VPRecipeValue defined by a VPSingleDefRecipe.
class VPSingleDefValue : public VPRecipeValue {
friend class VPDef;
friend class VPSingleDefRecipe;
@@ -482,6 +482,8 @@ class VPDef {
/// Add \p V as a defined value by this VPDef.
void addDefinedValue(VPRecipeValue *V) {
+ assert(V->isDefinedBy(this) &&
+ "can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
}
>From 43e8d80f36bb17ba03647f8dfbfd12378e743618 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 13 May 2026 11:45:06 +0100
Subject: [PATCH 3/6] [VPlan] Make VPWidenLoad(EVL)Recipe VPSingleDefs (NFC).
This updates VPWidenMemoryRecipe to no longer inherit directly from
VPRecipeBase. Instead VPWidenMemoryRecipe is turned into a mixin, like
VPIRMetadata and VPPhiAccessors. This in turn allows updating
VPWidenLoad(EVL)Recipe to inherit directly from VPSingleDefRecipe. This
brings them in line with all other recipes defining a single VPValue.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 132 +++++++++++-------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 23 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 32 +++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +-
5 files changed, 110 insertions(+), 83 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 7e0dc1371bcf1..f8434f80ab751 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -643,18 +643,16 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPWidenIntOrFpInductionSC:
case VPRecipeBase::VPWidenPointerInductionSC:
case VPRecipeBase::VPReductionPHISC:
+ case VPRecipeBase::VPWidenLoadEVLSC:
+ case VPRecipeBase::VPWidenLoadSC:
return true;
case VPRecipeBase::VPBranchOnMaskSC:
case VPRecipeBase::VPInterleaveEVLSC:
case VPRecipeBase::VPInterleaveSC:
case VPRecipeBase::VPIRInstructionSC:
- case VPRecipeBase::VPWidenLoadEVLSC:
- case VPRecipeBase::VPWidenLoadSC:
case VPRecipeBase::VPWidenStoreEVLSC:
case VPRecipeBase::VPWidenStoreSC:
case VPRecipeBase::VPHistogramSC:
- // TODO: Widened stores don't define a value, but widened loads do. Split
- // the recipes to be able to make widened loads VPSingleDefRecipes.
return false;
}
llvm_unreachable("Unhandled VPRecipeID");
@@ -3487,10 +3485,9 @@ class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
#endif
};
-/// A common base class for widening memory operations. An optional mask can be
+/// A common mixin class for widening memory operations. An optional mask can be
/// provided as the last operand.
-class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
- public VPIRMetadata {
+class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
protected:
Instruction &Ingredient;
@@ -3507,39 +3504,27 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
assert(!IsMasked && "cannot re-set mask");
if (!Mask)
return;
- addOperand(Mask);
+ getAsRecipe()->addOperand(Mask);
IsMasked = true;
}
- VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
- std::initializer_list<VPValue *> Operands,
- bool Consecutive, const VPIRMetadata &Metadata,
- DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
+ VPWidenMemoryRecipe(Instruction &I, bool Consecutive,
+ const VPIRMetadata &Metadata)
+ : VPIRMetadata(Metadata), Ingredient(I),
Alignment(getLoadStoreAlignment(&I)), Consecutive(Consecutive) {}
public:
- VPWidenMemoryRecipe *clone() override {
- llvm_unreachable("cloning not supported");
- }
-
- static inline bool classof(const VPRecipeBase *R) {
- return R->getVPRecipeID() == VPRecipeBase::VPWidenLoadSC ||
- R->getVPRecipeID() == VPRecipeBase::VPWidenStoreSC ||
- R->getVPRecipeID() == VPRecipeBase::VPWidenLoadEVLSC ||
- R->getVPRecipeID() == VPRecipeBase::VPWidenStoreEVLSC;
- }
+ virtual ~VPWidenMemoryRecipe() = default;
- static inline bool classof(const VPUser *U) {
- auto *R = dyn_cast<VPRecipeBase>(U);
- return R && classof(R);
- }
+ /// Return a VPRecipeBase* to the current object.
+ virtual VPRecipeBase *getAsRecipe() = 0;
+ virtual const VPRecipeBase *getAsRecipe() const = 0;
/// Return whether the loaded-from / stored-to addresses are consecutive.
bool isConsecutive() const { return Consecutive; }
/// Return the address accessed by this recipe.
- VPValue *getAddr() const { return getOperand(0); }
+ VPValue *getAddr() const { return getAsRecipe()->getOperand(0); }
/// Returns true if the recipe is masked.
bool isMasked() const { return IsMasked; }
@@ -3548,33 +3533,27 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// by a nullptr.
VPValue *getMask() const {
// Mask is optional and therefore the last operand.
- return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
+ const VPRecipeBase *R = getAsRecipe();
+ return isMasked() ? R->getOperand(R->getNumOperands() - 1) : nullptr;
}
/// Returns the alignment of the memory access.
Align getAlign() const { return Alignment; }
- /// Generate the wide load/store.
- void execute(VPTransformState &State) override {
- llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
- }
-
/// Return the cost of this VPWidenMemoryRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override;
+ InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
Instruction &getIngredient() const { return Ingredient; }
};
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
+ public VPWidenMemoryRecipe {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
- : VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
- Consecutive, Metadata, DL),
- VPRecipeValue(this, &Load) {
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenLoadSC, {Addr}, &Load, DL),
+ VPWidenMemoryRecipe(Load, Consecutive, Metadata) {
setMask(Mask);
}
@@ -3588,6 +3567,12 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// Generate a wide load or gather.
void execute(VPTransformState &State) override;
+ /// Return the cost of this VPWidenLoadRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override {
+ return VPWidenMemoryRecipe::computeCost(VF, Ctx);
+ }
+
/// Returns true if the recipe only uses the first lane of operand \p Op.
bool usesFirstLaneOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
@@ -3598,6 +3583,9 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
}
protected:
+ VPRecipeBase *getAsRecipe() override { return this; }
+ const VPRecipeBase *getAsRecipe() const override { return this; }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent,
@@ -3608,17 +3596,20 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// A recipe for widening load operations with vector-predication intrinsics,
/// using the address to load from, the explicit vector length and an optional
/// mask.
-struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct VPWidenLoadEVLRecipe final : public VPSingleDefRecipe,
+ public VPWidenMemoryRecipe {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
- : VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
- {Addr, &EVL}, L.isConsecutive(), L,
- L.getDebugLoc()),
- VPRecipeValue(this, &getIngredient()) {
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenLoadEVLSC, {Addr, &EVL},
+ &L.getIngredient(), L.getDebugLoc()),
+ VPWidenMemoryRecipe(L.getIngredient(), L.isConsecutive(), L) {
setMask(Mask);
}
+ VPWidenLoadEVLRecipe *clone() override {
+ llvm_unreachable("cloning not supported");
+ }
+
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadEVLSC)
/// Return the EVL operand.
@@ -3641,6 +3632,9 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
}
protected:
+ VPRecipeBase *getAsRecipe() override { return this; }
+ const VPRecipeBase *getAsRecipe() const override { return this; }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
@@ -3650,12 +3644,13 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
/// A recipe for widening store operations, using the stored value, the address
/// to store to and an optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
+struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPRecipeBase,
+ public VPWidenMemoryRecipe {
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
VPValue *Mask, bool Consecutive,
const VPIRMetadata &Metadata, DebugLoc DL)
- : VPWidenMemoryRecipe(VPRecipeBase::VPWidenStoreSC, Store,
- {Addr, StoredVal}, Consecutive, Metadata, DL) {
+ : VPRecipeBase(VPRecipeBase::VPWidenStoreSC, {Addr, StoredVal}, DL),
+ VPWidenMemoryRecipe(Store, Consecutive, Metadata) {
setMask(Mask);
}
@@ -3673,6 +3668,12 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
/// Generate a wide store or scatter.
void execute(VPTransformState &State) override;
+ /// Return the cost of this VPWidenStoreRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override {
+ return VPWidenMemoryRecipe::computeCost(VF, Ctx);
+ }
+
/// Returns true if the recipe only uses the first lane of operand \p Op.
bool usesFirstLaneOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
@@ -3683,6 +3684,9 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
}
protected:
+ VPRecipeBase *getAsRecipe() override { return this; }
+ const VPRecipeBase *getAsRecipe() const override { return this; }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent,
@@ -3693,15 +3697,20 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
/// A recipe for widening store operations with vector-predication intrinsics,
/// using the value to store, the address to store to, the explicit vector
/// length and an optional mask.
-struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
+struct VPWidenStoreEVLRecipe final : public VPRecipeBase,
+ public VPWidenMemoryRecipe {
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue *Addr,
VPValue *StoredVal, VPValue &EVL, VPValue *Mask)
- : VPWidenMemoryRecipe(VPRecipeBase::VPWidenStoreEVLSC, S.getIngredient(),
- {Addr, StoredVal, &EVL}, S.isConsecutive(), S,
- S.getDebugLoc()) {
+ : VPRecipeBase(VPRecipeBase::VPWidenStoreEVLSC, {Addr, StoredVal, &EVL},
+ S.getDebugLoc()),
+ VPWidenMemoryRecipe(S.getIngredient(), S.isConsecutive(), S) {
setMask(Mask);
}
+ VPWidenStoreEVLRecipe *clone() override {
+ llvm_unreachable("cloning not supported");
+ }
+
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStoreEVLSC)
/// Return the address accessed by this recipe.
@@ -3732,6 +3741,9 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
}
protected:
+ VPRecipeBase *getAsRecipe() override { return this; }
+ const VPRecipeBase *getAsRecipe() const override { return this; }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
@@ -4085,6 +4097,18 @@ struct CastInfo<VPPhiAccessors, VPRecipeBase>
: public ForwardToPointerCast<VPPhiAccessors, VPRecipeBase *,
CastInfo<VPPhiAccessors, VPRecipeBase *>> {};
+/// Support casting from VPRecipeBase / VPUser -> VPWidenMemoryRecipe.
+template <>
+struct CastInfo<VPWidenMemoryRecipe, VPRecipeBase *>
+ : vpdetail::CastInfoMixinImpl<VPWidenMemoryRecipe, VPWidenLoadRecipe,
+ VPWidenLoadEVLRecipe, VPWidenStoreRecipe,
+ VPWidenStoreEVLRecipe> {};
+template <>
+struct CastInfo<VPWidenMemoryRecipe, const VPRecipeBase *>
+ : public ConstStrippingForwardingCast<
+ VPWidenMemoryRecipe, const VPRecipeBase *,
+ CastInfo<VPWidenMemoryRecipe, VPRecipeBase *>> {};
+
/// Support casting from VPRecipeBase -> VPIRMetadata.
template <>
struct CastInfo<VPIRMetadata, VPRecipeBase *>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index adf5030dda159..6403f9657a51f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -201,7 +201,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenCallRecipe *R) {
}
Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R) {
- assert((isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R)) &&
+ assert((isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R->getAsRecipe())) &&
"Store recipes should not define any values");
return cast<LoadInst>(&R->getIngredient())->getType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 397366bae9af9..6bf484c90ee93 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3624,10 +3624,11 @@ void VPPredInstPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ const VPRecipeBase *R = getAsRecipe();
Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF);
unsigned AS = cast<PointerType>(Ctx.Types.inferScalarType(getAddr()))
->getAddressSpace();
- unsigned Opcode = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this)
+ unsigned Opcode = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R)
? Instruction::Load
: Instruction::Store;
@@ -3635,12 +3636,12 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
// TODO: Using the original IR may not be accurate.
// Currently, ARM will use the underlying IR to calculate gather/scatter
// instruction cost.
- [[maybe_unused]] auto IsReverseMask = [this]() {
+ [[maybe_unused]] auto IsReverseMask = [this, R]() {
VPValue *Mask = getMask();
if (!Mask)
return false;
- if (isa<VPWidenLoadEVLRecipe, VPWidenStoreEVLRecipe>(this))
+ if (isa<VPWidenLoadEVLRecipe, VPWidenStoreEVLRecipe>(R))
return match(Mask, m_Intrinsic<Intrinsic::experimental_vp_reverse>());
return match(Mask, m_Reverse(m_VPValue()));
@@ -3655,10 +3656,10 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
if (!vputils::isSingleScalar(getAddr()))
PtrTy = toVectorTy(PtrTy, VF);
- unsigned IID = isa<VPWidenLoadRecipe>(this) ? Intrinsic::masked_gather
- : isa<VPWidenStoreRecipe>(this) ? Intrinsic::masked_scatter
- : isa<VPWidenLoadEVLRecipe>(this) ? Intrinsic::vp_gather
- : Intrinsic::vp_scatter;
+ unsigned IID = isa<VPWidenLoadRecipe>(R) ? Intrinsic::masked_gather
+ : isa<VPWidenStoreRecipe>(R) ? Intrinsic::masked_scatter
+ : isa<VPWidenLoadEVLRecipe>(R) ? Intrinsic::vp_gather
+ : Intrinsic::vp_scatter;
return Ctx.TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,
Ctx.CostKind) +
Ctx.TTI.getMemIntrinsicInstrCost(
@@ -3669,14 +3670,14 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
InstructionCost Cost = 0;
if (IsMasked) {
- unsigned IID = isa<VPWidenLoadRecipe>(this) ? Intrinsic::masked_load
- : Intrinsic::masked_store;
+ unsigned IID = isa<VPWidenLoadRecipe>(R) ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
Cost += Ctx.TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(IID, Ty, Alignment, AS), Ctx.CostKind);
} else {
TTI::OperandValueInfo OpInfo = Ctx.getOperandInfo(
- isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(this) ? getOperand(0)
- : getOperand(1));
+ isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R) ? R->getOperand(0)
+ : R->getOperand(1));
Cost += Ctx.TTI.getMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind,
OpInfo, &Ingredient);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d18488ccf69db..bdac22844a78b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3555,10 +3555,11 @@ void VPlanTransforms::createInterleaveGroups(
for (VPBasicBlock *VPBB :
VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(
Plan.getVectorLoopRegion()->getEntryBasicBlock())))
- for (VPRecipeBase &R :
- make_filter_range(*VPBB, IsaPred<VPWidenMemoryRecipe>)) {
- auto &MemR = cast<VPWidenMemoryRecipe>(R);
- IRMemberToRecipe[&MemR.getIngredient()] = &MemR;
+ for (VPRecipeBase &R : make_filter_range(*VPBB, [](VPRecipeBase &R) {
+ return isa<VPWidenMemoryRecipe>(&R);
+ })) {
+ auto *MemR = cast<VPWidenMemoryRecipe>(&R);
+ IRMemberToRecipe[&MemR->getIngredient()] = MemR;
}
// Interleave memory: for each Interleave Group we marked earlier as relevant
@@ -3577,14 +3578,14 @@ void VPlanTransforms::createInterleaveGroups(
auto *Start = IRMemberToRecipe.lookup(IG->getMember(0));
VPIRMetadata InterleaveMD(*Start);
SmallVector<VPValue *, 4> StoredValues;
- if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Start))
+ if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Start->getAsRecipe()))
StoredValues.push_back(StoreR->getStoredValue());
for (unsigned I = 1; I < IG->getFactor(); ++I) {
Instruction *MemberI = IG->getMember(I);
if (!MemberI)
continue;
VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(MemberI);
- if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(MemoryR))
+ if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(MemoryR->getAsRecipe()))
StoredValues.push_back(StoreR->getStoredValue());
InterleaveMD.intersect(*MemoryR);
}
@@ -3595,6 +3596,7 @@ void VPlanTransforms::createInterleaveGroups(
Instruction *IRInsertPos = IG->getInsertPos();
auto *InsertPos = IRMemberToRecipe.lookup(IRInsertPos);
+ VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
GEPNoWrapFlags NW = GEPNoWrapFlags::none();
if (auto *Gep = dyn_cast<GetElementPtrInst>(
@@ -3604,7 +3606,7 @@ void VPlanTransforms::createInterleaveGroups(
// Get or create the start address for the interleave group.
VPValue *Addr = Start->getAddr();
VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
- if (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPos)) {
+ if (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPosR)) {
// We cannot re-use the address of member zero because it does not
// dominate the insert position. Instead, use the address of the insert
// position and create a PtrAdd adjusting it to the address of member
@@ -3619,7 +3621,7 @@ void VPlanTransforms::createInterleaveGroups(
IG->getIndex(IRInsertPos),
/*IsSigned=*/true);
VPValue *OffsetVPV = Plan.getConstantInt(-Offset);
- VPBuilder B(InsertPos);
+ VPBuilder B(InsertPosR);
Addr = B.createNoWrapPtrAdd(InsertPos->getAddr(), OffsetVPV, NW);
}
// If the group is reverse, adjust the index to refer to the last vector
@@ -3629,19 +3631,19 @@ void VPlanTransforms::createInterleaveGroups(
if (IG->isReverse()) {
auto *ReversePtr = new VPVectorEndPointerRecipe(
Addr, &Plan.getVF(), getLoadStoreType(IRInsertPos),
- -(int64_t)IG->getFactor(), NW, InsertPos->getDebugLoc());
- ReversePtr->insertBefore(InsertPos);
+ -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
+ ReversePtr->insertBefore(InsertPosR);
Addr = ReversePtr;
}
- auto *VPIG = new VPInterleaveRecipe(IG, Addr, StoredValues,
- InsertPos->getMask(), NeedsMaskForGaps,
- InterleaveMD, InsertPos->getDebugLoc());
- VPIG->insertBefore(InsertPos);
+ auto *VPIG = new VPInterleaveRecipe(
+ IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
+ InterleaveMD, InsertPosR->getDebugLoc());
+ VPIG->insertBefore(InsertPosR);
unsigned J = 0;
for (unsigned i = 0; i < IG->getFactor(); ++i)
if (Instruction *Member = IG->getMember(i)) {
- VPRecipeBase *MemberR = IRMemberToRecipe.lookup(Member);
+ VPRecipeBase *MemberR = IRMemberToRecipe.lookup(Member)->getAsRecipe();
if (!Member->getType()->isVoidTy()) {
VPValue *OriginalV = MemberR->getVPSingleValue();
OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 24adcea1040b5..0e380e0a6015c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -113,7 +113,7 @@ static bool poisonGuaranteesUB(const VPValue *V) {
for (VPUser *U : Current->users()) {
// Check if Current is used as an address operand for load/store.
- if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(U)) {
+ if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(U))) {
if (MemR->getAddr() == Current)
return true;
continue;
@@ -819,7 +819,7 @@ bool vputils::isUsedByLoadStoreAddress(const VPValue *V) {
RepR->getOperand(1) == Cur)
return true;
}
- if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(U)) {
+ if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(U))) {
if (MemR->getAddr() == Cur && MemR->isConsecutive())
return true;
}
>From 6efa24499c15ffd632670af6003270f32a1f763d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 14 May 2026 19:31:48 +0100
Subject: [PATCH 4/6] !fixup address comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 20 +++++++++++--------
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 19 +++++++++---------
.../Transforms/Vectorize/VPlanTest.cpp | 4 ++--
4 files changed, 25 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 07123504262e4..5ed84f6d43275 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -131,8 +131,8 @@ VPRecipeBase *VPValue::getDefiningRecipe() {
auto *DefValue = dyn_cast<VPRecipeValue>(this);
if (!DefValue)
return nullptr;
- if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(DefValue))
- return SV->getDef();
+ if (auto *MultiDef = dyn_cast<VPMultiDefValue>(DefValue))
+ return MultiDef->getDef();
return static_cast<VPSingleDefRecipe *>(DefValue);
}
@@ -146,7 +146,11 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPRecipeValue::~VPRecipeValue() = default;
+VPRecipeValue::~VPRecipeValue() {
+ assert(Users.empty() &&
+ "trying to delete a VPRecipeValue with remaining users");
+ getDefiningRecipe()->removeDefinedValue(this);
+}
VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
: VPRecipeValue(VPVSingleDefValueSC, UV) {
@@ -154,15 +158,15 @@ VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
Def->addDefinedValue(this);
}
-VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
+VPSingleDefValue::~VPSingleDefValue() {}
+
+VPMultiDefValue::VPMultiDefValue(VPRecipeBase *Def, Value *UV)
: VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
- assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
+ assert(Def && "VPMultiDefValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPStandaloneRecipeValue::~VPStandaloneRecipeValue() {
- Def->removeDefinedValue(this);
-}
+VPMultiDefValue::~VPMultiDefValue() {}
// Get the top-most entry block of \p Start. This is the entry block of the
// containing VPlan. This function is templated to support both const and non-const blocks
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 327446522cf13..f3841c8c6ba0d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -616,8 +616,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
- ~VPSingleDefRecipe() override { removeDefinedValue(this); }
-
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
case VPRecipeBase::VPDerivedIVSC:
@@ -2873,7 +2871,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPStandaloneRecipeValue(this, Inst);
+ new VPMultiDefValue(this, Inst);
}
} else {
for (auto *SV : StoredValues)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 849dfa2649da9..b962f267a96d6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -342,23 +342,24 @@ class VPSingleDefValue : public VPRecipeValue {
Value *UV = nullptr);
public:
+ ~VPSingleDefValue() override;
+
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVSingleDefValueSC;
}
};
-/// A VPRecipeValue that stores a pointer to its defining recipe.
-class VPStandaloneRecipeValue : public VPRecipeValue {
+/// A VPRecipeValue defined by a multi-def recipe, stores a pointer to it.
+class VPMultiDefValue : public VPRecipeValue {
friend class VPDef;
- /// Pointer to the VPRecipeBase that defines this VPValue.
+ /// Pointer to the multi-def recipe that defines this VPValue, among others.
VPRecipeBase *Def;
public:
- LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
- Value *UV = nullptr);
+ LLVM_ABI_FOR_TEST VPMultiDefValue(VPRecipeBase *Def, Value *UV = nullptr);
- ~VPStandaloneRecipeValue() override;
+ ~VPMultiDefValue() override;
VPRecipeBase *getDef() const { return Def; }
@@ -473,9 +474,9 @@ class VPUser {
/// Single-value VPDefs that also inherit from VPValue must make sure to inherit
/// from VPDef before VPValue.
class VPDef {
+ friend class VPRecipeValue;
friend class VPSingleDefValue;
- friend class VPSingleDefRecipe;
- friend class VPStandaloneRecipeValue;
+ friend class VPMultiDefValue;
/// The VPValues defined by this VPDef.
TinyPtrVector<VPRecipeValue *> DefinedValues;
@@ -495,7 +496,7 @@ class VPDef {
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
llvm::erase(DefinedValues, V);
- if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(V))
+ if (auto *SV = dyn_cast<VPMultiDefValue>(V))
SV->Def = nullptr;
}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 7cd7077b6116a..4b7993637c8cb 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1736,8 +1736,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPStandaloneRecipeValue(this);
- new VPStandaloneRecipeValue(this);
+ new VPMultiDefValue(this);
+ new VPMultiDefValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
>From e79fcb77367213f9833d2997dbb72ae67ba8a5c7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 16 May 2026 13:13:29 +0100
Subject: [PATCH 5/6] !fixup address comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 14 +++++---------
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 20 ++++++++++----------
3 files changed, 16 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 5ed84f6d43275..9b2813cffa3ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -128,12 +128,12 @@ bool VPRecipeValue::isDefinedBy(const VPDef *D) const {
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
- auto *DefValue = dyn_cast<VPRecipeValue>(this);
- if (!DefValue)
+ auto *RecipeValue = dyn_cast<VPRecipeValue>(this);
+ if (!RecipeValue)
return nullptr;
- if (auto *MultiDef = dyn_cast<VPMultiDefValue>(DefValue))
+ if (auto *MultiDef = dyn_cast<VPMultiDefValue>(RecipeValue))
return MultiDef->getDef();
- return static_cast<VPSingleDefRecipe *>(DefValue);
+ return static_cast<VPSingleDefRecipe *>(RecipeValue);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
@@ -158,16 +158,12 @@ VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
Def->addDefinedValue(this);
}
-VPSingleDefValue::~VPSingleDefValue() {}
-
VPMultiDefValue::VPMultiDefValue(VPRecipeBase *Def, Value *UV)
- : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+ : VPRecipeValue(VPVMultiDefValueSC, UV), Def(Def) {
assert(Def && "VPMultiDefValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPMultiDefValue::~VPMultiDefValue() {}
-
// Get the top-most entry block of \p Start. This is the entry block of the
// containing VPlan. This function is templated to support both const and non-const blocks
template <typename T> static T *getPlanEntry(T *Start) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index fa2c60ca60b26..1cb131eeab446 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -601,7 +601,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDefRecipe is a base class for recipes for modeling a sequence of one
/// or more output IR that define a single result VPValue. Note that
-/// VPRecipeBase must be inherited from before VPValue.
+/// VPSingleDefRecipe must inherit from VPRecipeBase before VPSingleDefValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index b962f267a96d6..e2263f8d52745 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -80,12 +80,12 @@ class LLVM_ABI_FOR_TEST VPValue {
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
- VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
- VPVStandaloneRecipeValueSC, /// A standalone VPValue defined by a recipe.
- VPVSingleDefValueSC, /// A VPValue embedded in a VPSingleDefRecipe.
- VPRegionValueSC, /// A VPValue sub-class that is defined by a
- /// region, like a loop region canonical IV.
+ VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
+ VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
+ VPVMultiDefValueSC, /// A VPValue defined by a multi-def recipe.
+ VPVSingleDefValueSC, /// A VPValue defined by a VPSingleDefRecipe.
+ VPRegionValueSC, /// A VPValue sub-class that is defined by a
+ /// region, like a loop region canonical IV.
};
VPValue(const VPValue &) = delete;
@@ -326,7 +326,7 @@ class VPRecipeValue : public VPValue {
LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
+ return V->getVPValueID() == VPVMultiDefValueSC ||
V->getVPValueID() == VPVSingleDefValueSC;
}
};
@@ -342,7 +342,7 @@ class VPSingleDefValue : public VPRecipeValue {
Value *UV = nullptr);
public:
- ~VPSingleDefValue() override;
+ ~VPSingleDefValue() override = default;
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVSingleDefValueSC;
@@ -359,12 +359,12 @@ class VPMultiDefValue : public VPRecipeValue {
public:
LLVM_ABI_FOR_TEST VPMultiDefValue(VPRecipeBase *Def, Value *UV = nullptr);
- ~VPMultiDefValue() override;
+ ~VPMultiDefValue() override = default;
VPRecipeBase *getDef() const { return Def; }
static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVStandaloneRecipeValueSC;
+ return V->getVPValueID() == VPVMultiDefValueSC;
}
};
>From 3f723bf624bc61fe0a8dbfd22dde9c42a75c005f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 16 May 2026 22:02:35 +0100
Subject: [PATCH 6/6] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1cb131eeab446..193e065fbfa77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -599,7 +599,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
return R->getVPRecipeID() == VPRecipeID; \
}
-/// VPSingleDefRecipe is a base class for recipes for modeling a sequence of one
+/// VPSingleDefRecipe is a base class for recipes that model a sequence of one
/// or more output IR that define a single result VPValue. Note that
/// VPSingleDefRecipe must inherit from VPRecipeBase before VPSingleDefValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index e2263f8d52745..d229935c407e1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -82,8 +82,8 @@ class LLVM_ABI_FOR_TEST VPValue {
enum {
VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
- VPVMultiDefValueSC, /// A VPValue defined by a multi-def recipe.
VPVSingleDefValueSC, /// A VPValue defined by a VPSingleDefRecipe.
+ VPVMultiDefValueSC, /// A VPValue defined by a multi-def recipe.
VPRegionValueSC, /// A VPValue sub-class that is defined by a
/// region, like a loop region canonical IV.
};
@@ -314,7 +314,7 @@ class VPRecipeValue : public VPValue {
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
- /// NOTE: Only used by VPDef to assert that VPRecipeValues removed from
+ /// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
/// /p D are associated with its VPRecipeBase.
bool isDefinedBy(const VPDef *D) const;
#endif
More information about the llvm-commits
mailing list