[llvm] [VPlan] Add type in VPRecipeValue, migrate first set of recipes. (NFC) (PR #195485)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat May 2 14:39:22 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/195485
This patch adds a new Type field to hold the scalar type for the
VPRecipeValue. It starts the migration of an initial set of recipes to
set the scalar type and use it directly in VPTypeAnalysis, removing
special handling for those types.
ventually, VPTypeAnalysis can be retired when the migration is
complete, hopefully eliminating the need for type related caching, which
can lead to subtle invalidation errors.
To help the transition, a temporary getScalarTypeOrInfer has been added,
which returns the scalar type set in the VPValue for already migrated
recipes or falls back o type inferrence otherwise.
VPlanVerifier has been extended verify the set scalar type where
possible.
Tracking the type directly means a few places that change the type will
need to be updated to create a new replacement recipe with correct
types.
Depends on https://github.com/llvm/llvm-project/pull/195483
>From e400e76b5df58b174c3215673ec0526967e8e865 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 11 Apr 2026 17:19:53 +0100
Subject: [PATCH 1/2] [VPlan] Remove Def pointer from VPRecipeValue for
single-def recipes
For VPSingleDefRecipe, the VPRecipeValue's Def pointer always points
back to the containing VPRecipeBase, which is computable via
static_cast.
Introduce 2 VPRecipeValue subclasses to distinguish the VPValues defined
by VPSingleDefRecipes (VPSingleDefValue), and VPStandaloneValue for
other recipes.
The former does not need to store a pointer to the defining recipe, as
it can be computed via static_cast. This saves 8 bytes for most recipes.
I plan to use the extra bytes to store the type directly in VPValue as
follow-ups.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 31 +++++---
llvm/lib/Transforms/Vectorize/VPlan.h | 22 +++---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 70 ++++++++++++++-----
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
4 files changed, 89 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 91fa4c57e5bf9..cf7ebf8c3d2a8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -122,17 +122,22 @@ void VPRecipeBase::dump() const {
#endif
#if !defined(NDEBUG)
-bool VPRecipeValue::isDefinedBy(const VPDef *D) const { return Def == D; }
+bool VPRecipeValue::isDefinedBy(const VPDef *D) const {
+ return getDefiningRecipe() == D;
+}
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ if (!DefValue)
+ return nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(DefValue))
+ return SV->getDef();
+ return static_cast<VPSingleDefRecipe *>(DefValue);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ return const_cast<VPValue *>(this)->getDefiningRecipe();
}
Value *VPValue::getLiveInIRValue() const {
@@ -141,15 +146,21 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPRecipeValue::VPRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPValue(VPVRecipeValueSC, UV), Def(Def) {
- assert(Def && "VPRecipeValue requires a defining recipe");
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
+ : VPRecipeValue(VPVSingleDefValueSC, UV) {
+ assert(Def && "VPSingleDefValue requires a defining recipe");
+ Def->addDefinedValue(this);
+}
+
+VPRecipeValue::~VPRecipeValue() = default;
+
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+ assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPRecipeValue::~VPRecipeValue() {
- assert(Users.empty() &&
- "trying to delete a VPRecipeValue with remaining users");
+VPStandaloneRecipeValue::~VPStandaloneRecipeValue() {
Def->removeDefinedValue(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 592d83af2295f..783d3a1f8d17d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -601,16 +601,17 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
-/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+
+ ~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -2869,7 +2870,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst);
}
} else {
for (auto *SV : StoredValues)
@@ -3576,13 +3577,14 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
+ : public VPWidenMemoryRecipe,
+ public VPStandaloneRecipeValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecutive, Metadata, DL),
- VPRecipeValue(this, &Load) {
+ VPStandaloneRecipeValue(this, &Load) {
setMask(Mask);
}
@@ -3617,13 +3619,13 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// using the address to load from, the explicit vector length and an optional
/// mask.
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+ public VPStandaloneRecipeValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L,
L.getDebugLoc()),
- VPRecipeValue(this, &getIngredient()) {
+ VPStandaloneRecipeValue(this, &getIngredient()) {
setMask(Mask);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8f9806adf774e..4cb911b8e74c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -39,6 +39,7 @@ class VPDef;
class VPSlotTracker;
class VPUser;
class VPRecipeBase;
+class VPSingleDefRecipe;
class VPPhiAccessors;
class VPRegionValue;
class VPRegionBlock;
@@ -79,11 +80,12 @@ class LLVM_ABI_FOR_TEST VPValue {
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
- VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
- VPVRecipeValueSC, /// A VPValue defined by a recipe.
- VPRegionValueSC, /// A VPValue sub-class that is defined by a region, like
- /// the canonical IV of a loop region.
+ VPVIRValueSC, /// A live-in VPValue wrapping an IR Value.
+ VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
+ VPVStandaloneRecipeValueSC, /// A standalone VPValue defined by a recipe.
+ VPVSingleDefValueSC, /// A VPValue embedded in a VPSingleDefRecipe.
+ VPRegionValueSC, /// A VPValue sub-class that is defined by a
+ /// region, like a loop region canonical IV.
};
VPValue(const VPValue &) = delete;
@@ -299,14 +301,11 @@ struct VPSymbolicValue : public VPValue {
bool Materialized = false;
};
-/// A VPValue defined by a recipe that produces one or more values.
+/// Abstract base class for VPValues defined by a VPRecipeBase.
class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
- /// Pointer to the VPRecipeBase that defines this VPValue.
- VPRecipeBase *Def;
-
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
/// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
@@ -314,13 +313,51 @@ class VPRecipeValue : public VPValue {
bool isDefinedBy(const VPDef *D) const;
#endif
+protected:
+ VPRecipeValue(unsigned char SC, Value *UV = nullptr) : VPValue(SC, UV) {}
+
+public:
+ LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
+ V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue embedded as a subobject of VPSingleDefRecipe.
+class VPSingleDefValue : public VPRecipeValue {
+ friend class VPDef;
+ friend class VPSingleDefRecipe;
+
+protected:
+ /// Construct a VPSingleDefValue. Must only be used by VPSingleDefRecipe.
+ LLVM_ABI_FOR_TEST VPSingleDefValue(VPSingleDefRecipe *Def,
+ Value *UV = nullptr);
+
+public:
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSingleDefValueSC;
+ }
+};
+
+/// A VPRecipeValue that stores a pointer to its defining recipe.
+class VPStandaloneRecipeValue : public VPRecipeValue {
+ friend class VPDef;
+
+ /// Pointer to the VPRecipeBase that defines this VPValue.
+ VPRecipeBase *Def;
+
public:
- LLVM_ABI_FOR_TEST VPRecipeValue(VPRecipeBase *Def, Value *UV = nullptr);
+ LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
+ Value *UV = nullptr);
- LLVM_ABI_FOR_TEST virtual ~VPRecipeValue();
+ ~VPStandaloneRecipeValue() override;
+
+ VPRecipeBase *getDef() const { return Def; }
static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVRecipeValueSC;
+ return V->getVPValueID() == VPVStandaloneRecipeValueSC;
}
};
@@ -430,15 +467,15 @@ class VPUser {
/// Single-value VPDefs that also inherit from VPValue must make sure to inherit
/// from VPDef before VPValue.
class VPDef {
- friend class VPRecipeValue;
+ friend class VPSingleDefValue;
+ friend class VPSingleDefRecipe;
+ friend class VPStandaloneRecipeValue;
/// The VPValues defined by this VPDef.
TinyPtrVector<VPRecipeValue *> DefinedValues;
/// Add \p V as a defined value by this VPDef.
void addDefinedValue(VPRecipeValue *V) {
- assert(V->isDefinedBy(this) &&
- "can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
}
@@ -450,7 +487,8 @@ class VPDef {
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
llvm::erase(DefinedValues, V);
- V->Def = nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(V))
+ SV->Def = nullptr;
}
public:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a1ddda7eda969..9cc1d90d6256d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1698,8 +1698,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPRecipeValue(this);
- new VPRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
+ new VPStandaloneRecipeValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
>From 43c026ffb7369e5b1051aa5cc675701b47cd2d20 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 1 May 2026 14:18:29 +0100
Subject: [PATCH 2/2] [VPlan] Add type in VPRecipeValue, migrate first set of
recipes. (NFC)
This patch adds a new Type field to hold the scalar type for the
VPRecipeValue. It starts the migration of an initial set of recipes to
set the scalar type and use it directly in VPTypeAnalysis, removing
special handling for those types.
ventually, VPTypeAnalysis can be retired when the migration is
complete, hopefully eliminating the need for type related caching, which
can lead to subtle invalidation errors.
To help the transition, a temporary getScalarTypeOrInfer has been added,
which returns the scalar type set in the VPValue for already migrated
recipes or falls back o type inferrence otherwise.
VPlanVerifier has been extended verify the set scalar type where
possible.
Tracking the type directly means a few places that change the type will
need to be updated to create a new replacement recipe with correct
types.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 22 ++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 92 +++++++++++--------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 44 +++------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 4 -
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 18 +++-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 18 +++-
.../Transforms/Vectorize/VPlanVerifier.cpp | 56 +++++++++++
.../Transforms/Vectorize/VPlanTest.cpp | 7 +-
8 files changed, 174 insertions(+), 87 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index cf7ebf8c3d2a8..6834d4e86e2dd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -146,16 +146,30 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV)
- : VPRecipeValue(VPVSingleDefValueSC, UV) {
+Type *VPValue::getScalarType() const {
+ switch (getVPValueID()) {
+ case VPVIRValueSC:
+ return getUnderlyingValue()->getType();
+ case VPRegionValueSC:
+ return cast<VPRegionValue>(this)->getType();
+ case VPVStandaloneRecipeValueSC:
+ case VPVSingleDefValueSC:
+ return cast<VPRecipeValue>(this)->getScalarType();
+ }
+ llvm_unreachable("Unhandled VPValue subclass");
+}
+
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV, Type *Ty)
+ : VPRecipeValue(VPVSingleDefValueSC, UV, Ty) {
assert(Def && "VPSingleDefValue requires a defining recipe");
Def->addDefinedValue(this);
}
VPRecipeValue::~VPRecipeValue() = default;
-VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPRecipeValue(VPVStandaloneRecipeValueSC, UV), Def(Def) {
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV,
+ Type *Ty)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV, Ty), Def(Def) {
assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 783d3a1f8d17d..3245ed86a2852 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -599,6 +599,11 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
return R->getVPRecipeID() == VPRecipeID; \
}
+/// Return the scalar type of \p V. If \p V's scalar type has not been set
+/// (e.g. the defining recipe has not yet been migrated), fall back to
+/// VPTypeAnalysis using the plan of the defining recipe.
+LLVM_ABI Type *getScalarTypeOrInfer(VPValue *V);
+
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
@@ -611,6 +616,11 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+ VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Type *Ty, Value *UV = nullptr,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV, Ty) {}
+
~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
@@ -1113,6 +1123,12 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
DebugLoc DL = DebugLoc::getUnknown())
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
+ VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, const VPIRFlags &Flags,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPSingleDefRecipe(SC, Operands, ResultTy, /*UV=*/nullptr, DL),
+ VPIRFlags(Flags) {}
+
static inline bool classof(const VPRecipeBase *R) {
return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
R->getVPRecipeID() == VPRecipeBase::VPInstructionSC ||
@@ -1834,16 +1850,14 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Cast instruction opcode.
Instruction::CastOps Opcode;
- /// Result type for the cast.
- Type *ResultTy;
-
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, ResultTy, Flags,
+ DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
assert(hasRequiredFlagsForOpcode(Opcode) &&
@@ -1854,7 +1868,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
+ return new VPWidenCastRecipe(Opcode, getOperand(0), getScalarType(),
cast_or_null<CastInst>(getUnderlyingValue()),
*this, *this, getDebugLoc());
}
@@ -1871,7 +1885,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
Instruction::CastOps getOpcode() const { return Opcode; }
/// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
+ Type *getResultType() const { return getScalarType(); }
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1886,9 +1900,6 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
- /// Scalar return type of the intrinsic.
- Type *ResultTy;
-
/// True if the intrinsic may read from memory.
bool MayReadFromMemory;
@@ -1904,9 +1915,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments, Ty,
Flags, DL),
- VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {
@@ -1918,10 +1929,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments, Ty,
Flags, DL),
- VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
- ResultTy(Ty) {
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1937,10 +1947,11 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
- operands(), ResultTy, *this, *this,
- getDebugLoc());
- return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
- *this, *this, getDebugLoc());
+ operands(), getResultType(), *this,
+ *this, getDebugLoc());
+ return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(),
+ getResultType(), *this, *this,
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntrinsicSC)
@@ -1956,7 +1967,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
/// Return the scalar return type of the intrinsic.
- Type *getResultType() const { return ResultTy; }
+ Type *getResultType() const { return getScalarType(); }
/// Return to name of the intrinsic as string.
StringRef getIntrinsicName() const;
@@ -1993,8 +2004,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments,
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, CallArguments, Flags,
- DL),
+ : VPRecipeWithIRFlags(
+ VPRecipeBase::VPWidenCallSC, CallArguments,
+ cast<Function>(CallArguments.back()->getLiveInIRValue())
+ ->getReturnType(),
+ Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
@@ -2095,7 +2109,8 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands,
+ getScalarTypeOrInfer(Operands[0]), Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
@@ -2155,7 +2170,7 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *SourceElementTy,
int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
: VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC, {Ptr, VF},
- GEPFlags, DL),
+ getScalarTypeOrInfer(Ptr), GEPFlags, DL),
SourceElementTy(SourceElementTy), Stride(Stride) {
assert(Stride < 0 && "Stride must be negative");
}
@@ -2223,7 +2238,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr,
+ getScalarTypeOrInfer(Ptr), GEPFlags, DL),
SourceElementTy(SourceElementTy) {}
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
@@ -2582,11 +2598,13 @@ class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
std::string Name;
public:
- /// Create a new VPWidenPHIRecipe with incoming values \p IncomingvValues,
+ /// Create a new VPWidenPHIRecipe with incoming values \p IncomingValues,
/// debug location \p DL and \p Name.
VPWidenPHIRecipe(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues, DL),
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues,
+ getScalarTypeOrInfer(IncomingValues[0]),
+ /*UV=*/nullptr, DL),
Name(Name.str()) {}
VPWidenPHIRecipe *clone() override {
@@ -2708,7 +2726,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags {
VPReductionPHIRecipe *clone() override {
return new VPReductionPHIRecipe(
dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
- *getOperand(0), *getBackedgeValue(), Style, *this,
+ *getStartValue(), *getBackedgeValue(), Style, *this,
HasUsesOutsideReductionChain);
}
@@ -2870,7 +2888,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPStandaloneRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst, Inst->getType());
}
} else {
for (auto *SV : StoredValues)
@@ -3461,7 +3479,8 @@ class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
- : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV, DL) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV,
+ getScalarTypeOrInfer(PredV), nullptr, DL) {}
~VPPredInstPHIRecipe() override = default;
VPPredInstPHIRecipe *clone() override {
@@ -3584,7 +3603,7 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecutive, Metadata, DL),
- VPStandaloneRecipeValue(this, &Load) {
+ VPStandaloneRecipeValue(this, &Load, Load.getType()) {
setMask(Mask);
}
@@ -3625,7 +3644,8 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L,
L.getDebugLoc()),
- VPStandaloneRecipeValue(this, &getIngredient()) {
+ VPStandaloneRecipeValue(this, &getIngredient(),
+ getIngredient().getType()) {
setMask(Mask);
}
@@ -3754,8 +3774,7 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
const SCEV *Expr;
public:
- VPExpandSCEVRecipe(const SCEV *Expr)
- : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}), Expr(Expr) {}
+ VPExpandSCEVRecipe(const SCEV *Expr);
~VPExpandSCEVRecipe() override = default;
@@ -3864,7 +3883,8 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
public VPUnrollPartAccessor<1> {
public:
VPWidenCanonicalIVRecipe(VPRegionValue *CanonicalIV)
- : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV}) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV},
+ CanonicalIV->getType(), /*UV=*/nullptr) {}
~VPWidenCanonicalIVRecipe() override = default;
@@ -3981,7 +4001,7 @@ class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
Instruction::BinaryOps Opcode, FastMathFlags FMFs,
DebugLoc DL)
: VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC, {IV, Step, VF},
- FMFs, DL),
+ getScalarTypeOrInfer(IV), FMFs, DL),
InductionOpcode(Opcode) {}
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 5697856559bc2..1367bd99131f4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -204,17 +204,6 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenRecipe *R) {
llvm_unreachable("Unhandled opcode!");
}
-Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenCallRecipe *R) {
- auto &CI = *cast<CallInst>(R->getUnderlyingInstr());
- return CI.getType();
-}
-
-Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R) {
- assert((isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(R)) &&
- "Store recipes should not define any values");
- return cast<LoadInst>(&R->getIngredient())->getType();
-}
-
Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
unsigned Opcode = R->getUnderlyingInstr()->getOpcode();
@@ -275,17 +264,21 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
- if (auto *IRV = dyn_cast<VPIRValue>(V))
- return IRV->getType();
-
if (isa<VPSymbolicValue>(V)) {
// All VPValues without any underlying IR value (like the vector trip count
// or the backedge-taken count) have the same type as the canonical IV.
return CanonicalIVTy;
}
- if (auto *RegionV = dyn_cast<VPRegionValue>(V))
- return RegionV->getType();
+ if (isa<VPIRValue, VPRegionValue, VPStandaloneRecipeValue, VPExpandSCEVRecipe,
+ VPWidenPHIRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe,
+ VPWidenCanonicalIVRecipe, VPWidenCastRecipe, VPWidenIntrinsicRecipe,
+ VPWidenGEPRecipe, VPVectorPointerRecipe, VPVectorEndPointerRecipe,
+ VPWidenCallRecipe>(V)) {
+ Type *Ty = V->getScalarType();
+ assert(Ty && "Scalar type must be set by recipe construction");
+ return Ty;
+ }
Type *ResultTy =
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
@@ -300,26 +293,11 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
})
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
- .Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
- VPScalarIVStepsRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
- VPVectorEndPointerRecipe, VPWidenCanonicalIVRecipe>(
- [this](const VPRecipeBase *R) {
- return inferScalarType(R->getOperand(0));
- })
// VPInstructionWithType must be handled before VPInstruction.
- .Case<VPInstructionWithType, VPWidenIntrinsicRecipe,
- VPWidenCastRecipe>(
+ .Case<VPInstructionWithType>(
[](const auto *R) { return R->getResultType(); })
- .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
- VPWidenCallRecipe, VPWidenMemoryRecipe>(
+ .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe>(
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
- .Case([V](const VPInterleaveBase *R) {
- // TODO: Use info from interleave group.
- return V->getUnderlyingValue()->getType();
- })
- .Case([](const VPExpandSCEVRecipe *R) {
- return R->getSCEV()->getType();
- })
.Case([this](const VPReductionRecipe *R) {
return inferScalarType(R->getChainOp());
})
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index c1c9075420d1c..a14e05d9418f6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -23,8 +23,6 @@ class VPValue;
class VPBlendRecipe;
class VPInstruction;
class VPWidenRecipe;
-class VPWidenCallRecipe;
-class VPWidenMemoryRecipe;
class VPReplicateRecipe;
class VPRecipeBase;
class VPlan;
@@ -54,9 +52,7 @@ class VPTypeAnalysis {
Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
Type *inferScalarTypeForRecipe(const VPInstruction *R);
- Type *inferScalarTypeForRecipe(const VPWidenCallRecipe *R);
Type *inferScalarTypeForRecipe(const VPWidenRecipe *R);
- Type *inferScalarTypeForRecipe(const VPWidenMemoryRecipe *R);
Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
public:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ef217b92640c4..e4208e545c435 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -423,6 +423,20 @@ template class VPUnrollPartAccessor<2>;
template class VPUnrollPartAccessor<3>;
}
+Type *llvm::getScalarTypeOrInfer(VPValue *V) {
+ if (Type *Ty = V->getScalarType())
+ return Ty;
+ auto *Recipe = V->getDefiningRecipe();
+ assert(Recipe && Recipe->getParent() &&
+ "operand without scalar type must be a recipe in a plan");
+ VPTypeAnalysis TypeInfo(*Recipe->getParent()->getPlan());
+ return TypeInfo.inferScalarType(V);
+}
+
+VPExpandSCEVRecipe::VPExpandSCEVRecipe(const SCEV *Expr)
+ : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}, Expr->getType()),
+ Expr(Expr) {}
+
VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &MD,
DebugLoc DL, const Twine &Name)
@@ -2000,7 +2014,7 @@ bool VPWidenIntrinsicRecipe::usesFirstLaneOnly(const VPValue *Op) const {
void VPWidenIntrinsicRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN-INTRINSIC ";
- if (ResultTy->isVoidTy()) {
+ if (getResultType()->isVoidTy()) {
O << "void ";
} else {
printAsOperand(O, SlotTracker);
@@ -2945,7 +2959,7 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
- : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {}, {}),
+ : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {}),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 4cb911b8e74c5..0cc135bbecfb4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -194,6 +194,10 @@ class LLVM_ABI_FOR_TEST VPValue {
VPRecipeBase *getDefiningRecipe();
const VPRecipeBase *getDefiningRecipe() const;
+ /// Returns the scalar type of this VPValue, dispatching based on the
+ /// concrete subclass.
+ Type *getScalarType() const;
+
/// Returns true if this VPValue is defined by a recipe.
bool hasDefiningRecipe() const { return getDefiningRecipe(); }
@@ -306,6 +310,9 @@ class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
+ /// The scalar type of the value produced by this recipe.
+ Type *Ty = nullptr;
+
#if !defined(NDEBUG)
/// Returns true if this VPRecipeValue is defined by \p D.
/// NOTE: Only used by VPDef to assert that VPRecipeValues added/removed from
@@ -314,11 +321,15 @@ class VPRecipeValue : public VPValue {
#endif
protected:
- VPRecipeValue(unsigned char SC, Value *UV = nullptr) : VPValue(SC, UV) {}
+ VPRecipeValue(unsigned char SC, Value *UV = nullptr, Type *Ty = nullptr)
+ : VPValue(SC, UV), Ty(Ty) {}
public:
LLVM_ABI_FOR_TEST virtual ~VPRecipeValue() = 0;
+ /// Returns the scalar type of this VPRecipeValue.
+ Type *getScalarType() const { return Ty; }
+
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVStandaloneRecipeValueSC ||
V->getVPValueID() == VPVSingleDefValueSC;
@@ -333,7 +344,7 @@ class VPSingleDefValue : public VPRecipeValue {
protected:
/// Construct a VPSingleDefValue. Must only be used by VPSingleDefRecipe.
LLVM_ABI_FOR_TEST VPSingleDefValue(VPSingleDefRecipe *Def,
- Value *UV = nullptr);
+ Value *UV = nullptr, Type *Ty = nullptr);
public:
static bool classof(const VPValue *V) {
@@ -350,7 +361,8 @@ class VPStandaloneRecipeValue : public VPRecipeValue {
public:
LLVM_ABI_FOR_TEST VPStandaloneRecipeValue(VPRecipeBase *Def,
- Value *UV = nullptr);
+ Value *UV = nullptr,
+ Type *Ty = nullptr);
~VPStandaloneRecipeValue() override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 4b99829a21817..9396ea95e1922 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -42,6 +42,12 @@ class VPlanVerifier {
/// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask.
bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const;
+ /// Verify that the stored scalar type of \p R is consistent with the types
+ /// derived from its operands. A null stored type is tolerated during the
+ /// transition to fully threaded scalar types; once set, it must agree with
+ /// the operand-derived type.
+ bool verifyRecipeTypes(const VPRecipeBase &R) const;
+
bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
bool verifyBlock(const VPBlockBase *VPB);
@@ -199,6 +205,54 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
return true;
}
+bool VPlanVerifier::verifyRecipeTypes(const VPRecipeBase &R) const {
+ const auto *SR = dyn_cast<VPSingleDefRecipe>(&R);
+ if (!SR)
+ return true;
+
+ auto CheckScalarType = [&](Type *Derived) -> bool {
+ if (Derived == SR->getScalarType())
+ return true;
+ errs() << "Recipe result type does not match type derived from operands";
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ errs() << ": ";
+ R.dump();
+#endif
+ errs() << "\n";
+ return false;
+ };
+
+ auto CheckOperandTypes = [&]() -> bool {
+ if (all_of(drop_begin(R.operands()), [&R](VPValue *Op) {
+ return getScalarTypeOrInfer(R.getOperand(0)) ==
+ getScalarTypeOrInfer(Op);
+ }))
+ return true;
+ errs() << "Recipe operand types do not match";
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ errs() << ": ";
+ R.dump();
+#endif
+ errs() << "\n";
+ return false;
+ };
+
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return CheckScalarType(getScalarTypeOrInfer(R.getOperand(0)));
+ case VPRecipeBase::VPWidenPHISC:
+ case VPRecipeBase::VPPredInstPHISC:
+ return CheckOperandTypes() &&
+ CheckScalarType(getScalarTypeOrInfer(R.getOperand(0)));
+ default:
+ return true;
+ }
+ llvm_unreachable("all recipes must be handled above");
+}
+
bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!verifyPhiRecipes(VPBB))
return false;
@@ -219,6 +273,8 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
errs() << "not in a VPIRBasicBlock!\n";
return false;
}
+ if (!verifyRecipeTypes(R))
+ return false;
for (const VPValue *V : R.definedValues()) {
// Verify that we can infer a scalar type for each defined value. With
// assertions enabled, inferScalarType will perform some consistency
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 9cc1d90d6256d..0dbee19675a54 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -721,10 +721,9 @@ TEST_F(VPBasicBlockTest, reassociateBlocks) {
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("VPBB2");
VPBlockUtils::connectBlocks(VPBB1, VPBB2);
- auto *WidenPhi = new VPWidenPHIRecipe({});
IntegerType *Int32 = IntegerType::get(C, 32);
VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- WidenPhi->addOperand(Val);
+ auto *WidenPhi = new VPWidenPHIRecipe(ArrayRef<VPValue *>{Val});
VPBB2->appendRecipe(WidenPhi);
VPBasicBlock *VPBBNew = Plan.createVPBasicBlock("VPBBNew");
@@ -743,11 +742,9 @@ TEST_F(VPBasicBlockTest, reassociateBlocks) {
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
- auto *WidenPhi = new VPWidenPHIRecipe({});
IntegerType *Int32 = IntegerType::get(C, 32);
VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- WidenPhi->addOperand(Val);
- WidenPhi->addOperand(Val);
+ auto *WidenPhi = new VPWidenPHIRecipe({Val, Val});
VPBB2->appendRecipe(WidenPhi);
VPBasicBlock *VPBBNew = Plan.createVPBasicBlock("VPBBNew");
More information about the llvm-commits
mailing list