[llvm] [VPlan] Add specialized VPValue subclasses for different types (NFC) (PR #172758)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Dec 28 14:33:47 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/172758
>From acd59977e79acea1584ea0af793e5b4ec67451ab Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 17 Dec 2025 23:21:11 +0000
Subject: [PATCH 1/4] [VPlan] Add specialized VPValue subclasses for different
types (NFC)
This patch adds VPValue sub-classes for the different cases we currently
have:
* VPLiveIn: A live-in VPValue that wraps an underlying IR value
* VPSymbolicValue: A symbolic VPValue not tied to an underlying value,
e.g. the vector trip count or VF VPValues
* VPDefValue: A VPValue defined by a VPDef.
This has multiple benefits:
* clearer constructors for each kind of VPValue
* limited scope: for example allows moving VPDef member to VPDefValue,
reducing size of other VPValues.
* stricter type checking for member variables (e.g. using VPLiveIn in
the Value -> live-in map in VPlan, or using VPSymbolicValue for
symbolic member VPValues)
There probably are additional opportunities for cleanups as follow-ups.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 37 ++++---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 69 +++++++------
llvm/lib/Transforms/Vectorize/VPlan.h | 88 +++++++++--------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 19 ++--
.../Transforms/Vectorize/VPlanPatternMatch.h | 23 ++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 44 +++++----
.../Transforms/Vectorize/VPlanTransforms.cpp | 35 ++++---
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 99 ++++++++++++-------
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
11 files changed, 238 insertions(+), 190 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f8ee1484fb2ef..459553dc1134b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4073,9 +4073,11 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
} else {
auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
Function *CalledFn =
- WidenCall ? WidenCall->getCalledScalarFunction()
- : cast<Function>(R->getOperand(R->getNumOperands() - 1)
- ->getLiveInIRValue());
+ WidenCall
+ ? WidenCall->getCalledScalarFunction()
+ : cast<Function>(
+ cast<VPLiveIn>(R->getOperand(R->getNumOperands() - 1))
+ ->getValue());
Name = CalledFn->getName();
}
OS << " call to " << Name;
@@ -4276,7 +4278,8 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
}
case VPInstruction::ActiveLaneMask: {
unsigned Multiplier =
- cast<ConstantInt>(VPI->getOperand(2)->getLiveInIRValue())
+ cast<ConstantInt>(
+ cast<VPLiveIn>(VPI->getOperand(2))->getValue())
->getZExtValue();
C += VPI->cost(VF * Multiplier, CostCtx);
break;
@@ -7281,7 +7284,7 @@ static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
"RdxResult must be ComputeFindIVResult");
VPValue *StartVPV = RdxResult->getOperand(1);
match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
- return StartVPV->getLiveInIRValue();
+ return cast<VPLiveIn>(StartVPV)->getValue();
}
// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
@@ -7315,7 +7318,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
[[maybe_unused]] Value *StartV =
- EpiRedResult->getOperand(1)->getLiveInIRValue();
+ cast<VPLiveIn>(EpiRedResult->getOperand(1))->getValue();
auto *Cmp = cast<ICmpInst>(MainResumeValue);
assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
"AnyOf expected to start with ICMP_NE");
@@ -7325,7 +7328,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = Cmp->getOperand(0);
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
Value *StartV = getStartValueFromReductionResult(EpiRedResult);
- Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
+ Value *SentinelV = cast<VPLiveIn>(EpiRedResult->getOperand(2))->getValue();
using namespace llvm::PatternMatch;
Value *Cmp, *OrigResumeV, *CmpOp;
[[maybe_unused]] bool IsExpectedPattern =
@@ -7419,9 +7422,10 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// making any changes to the CFG.
DenseMap<const SCEV *, Value *> ExpandedSCEVs =
VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
- if (!ILV.getTripCount())
- ILV.setTripCount(BestVPlan.getTripCount()->getLiveInIRValue());
- else
+ if (!ILV.getTripCount()) {
+ // After expandSCEVs, TripCount is always a VPLiveIn.
+ ILV.setTripCount(cast<VPLiveIn>(BestVPlan.getTripCount())->getValue());
+ } else
assert(VectorizingEpilogue && "should only re-use the existing trip "
"count during epilogue vectorization");
@@ -9057,8 +9061,8 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) {
Value *Step = State.get(getStepValue(), VPLane(0));
Value *Index = State.get(getOperand(1), VPLane(0));
Value *DerivedIV = emitTransformedIndex(
- State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
- cast_if_present<BinaryOperator>(FPBinOp));
+ State.Builder, Index, cast<VPLiveIn>(getStartValue())->getValue(), Step,
+ Kind, cast_if_present<BinaryOperator>(FPBinOp));
DerivedIV->setName(Name);
State.set(this, DerivedIV, VPLane(0));
}
@@ -9406,7 +9410,8 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
continue;
VPValue *OrigStart = VPI->getOperand(1);
- if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
+ if (isGuaranteedNotToBeUndefOrPoison(
+ cast<VPLiveIn>(OrigStart)->getValue()))
continue;
VPInstruction *Freeze =
Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
@@ -9529,7 +9534,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
->getIncomingValueForBlock(L->getLoopPreheader());
RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
- Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
+ Value *StartV = cast<VPLiveIn>(RdxResult->getOperand(1))->getValue();
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
@@ -9554,7 +9559,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
if (auto *I = dyn_cast<Instruction>(Cmp))
InstsToMove.push_back(I);
- Value *Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
+ Value *Sentinel = cast<VPLiveIn>(RdxResult->getOperand(2))->getValue();
ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
if (auto *I = dyn_cast<Instruction>(ResumeV))
InstsToMove.push_back(I);
@@ -9592,7 +9597,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
auto *VPI = dyn_cast<VPInstruction>(&R);
if (VPI && VPI->getOpcode() == Instruction::Freeze) {
VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
- ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
+ ToFrozen.lookup(cast<VPLiveIn>(VPI->getOperand(0))->getValue())));
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 59550d9237e8f..7d6c694d10db9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -78,6 +78,8 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const VPRecipeBase &R) {
}
#endif
+Type *VPLiveIn::getType() const { return getUnderlyingValue()->getType(); }
+
Value *VPLane::getAsRuntimeExpr(IRBuilderBase &Builder,
const ElementCount &VF) const {
switch (LaneKind) {
@@ -92,15 +94,10 @@ Value *VPLane::getAsRuntimeExpr(IRBuilderBase &Builder,
}
VPValue::VPValue(const unsigned char SC, Value *UV, VPDef *Def)
- : SubclassID(SC), UnderlyingVal(UV), Def(Def) {
- if (Def)
- Def->addDefinedValue(this);
-}
+ : SubclassID(SC), UnderlyingVal(UV) {}
VPValue::~VPValue() {
assert(Users.empty() && "trying to delete a VPValue with remaining users");
- if (VPDef *Def = getDefiningRecipe())
- Def->removeDefinedValue(this);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -129,11 +126,29 @@ void VPDef::dump() const {
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
- return cast_or_null<VPRecipeBase>(Def);
+ auto *Def = dyn_cast<VPDefValue>(this);
+ if (!Def)
+ return nullptr;
+ return cast<VPRecipeBase>(Def->Def);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- return cast_or_null<VPRecipeBase>(Def);
+ auto *Def = dyn_cast<VPDefValue>(this);
+ if (!Def)
+ return nullptr;
+ return cast<VPRecipeBase>(Def->Def);
+}
+
+VPDefValue::VPDefValue(VPDef *Def, Value *UV)
+ : VPValue(VPVDefValueSC, UV, nullptr), Def(Def) {
+ assert(Def && "VPDefValue requires a defining recipe");
+ Def->addDefinedValue(this);
+}
+
+VPDefValue::~VPDefValue() {
+ assert(Users.empty() && "trying to delete a VPValue with remaining users");
+ if (Def)
+ Def->removeDefinedValue(this);
}
// Get the top-most entry block of \p Start. This is the entry block of the
@@ -229,8 +244,8 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
CurrentParentLoop(CurrentParentLoop), TypeAnalysis(*Plan), VPDT(*Plan) {}
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
- if (Def->isLiveIn())
- return Def->getLiveInIRValue();
+ if (isa<VPLiveIn, VPSymbolicValue>(Def))
+ return Def->getUnderlyingValue();
if (hasScalarValue(Def, Lane))
return Data.VPV2Scalars[Def][Lane.mapToCacheIndex(VF)];
@@ -262,8 +277,8 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
if (NeedsScalar) {
- assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def) ||
- !vputils::onlyFirstLaneUsed(Def) ||
+ assert((VF.isScalar() || isa<VPLiveIn, VPSymbolicValue>(Def) ||
+ hasVectorValue(Def) || !vputils::onlyFirstLaneUsed(Def) ||
(hasScalarValue(Def, VPLane(0)) &&
Data.VPV2Scalars[Def].size() == 1)) &&
"Trying to access a single scalar per part but has multiple scalars "
@@ -284,8 +299,8 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
};
if (!hasScalarValue(Def, {0})) {
- assert(Def->isLiveIn() && "expected a live-in");
- Value *IRV = Def->getLiveInIRValue();
+ assert((isa<VPLiveIn, VPSymbolicValue>(Def)) && "expected a live-in");
+ Value *IRV = Def->getUnderlyingValue();
Value *B = GetBroadcastInstrs(IRV);
set(Def, B);
return B;
@@ -865,7 +880,7 @@ VPlan::VPlan(Loop *L) {
}
VPlan::~VPlan() {
- VPValue DummyValue;
+ VPSymbolicValue DummyValue;
for (auto *VPB : CreatedBlocks) {
if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB)) {
@@ -1052,7 +1067,7 @@ void VPlan::printLiveIns(raw_ostream &O) const {
O << "\n";
if (TripCount) {
- if (TripCount->isLiveIn())
+ if (isa<VPLiveIn>(TripCount))
O << "Live-in ";
TripCount->printAsOperand(O, SlotTracker);
O << " = original trip-count";
@@ -1168,20 +1183,18 @@ VPlan *VPlan::duplicate() {
// Create VPlan, clone live-ins and remap operands in the cloned blocks.
auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
- for (VPValue *OldLiveIn : getLiveIns()) {
- Old2NewVPValues[OldLiveIn] =
- NewPlan->getOrAddLiveIn(OldLiveIn->getLiveInIRValue());
+ for (VPLiveIn *OldLiveIn : getLiveIns()) {
+ Old2NewVPValues[OldLiveIn] = NewPlan->getOrAddLiveIn(OldLiveIn->getValue());
}
Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount;
Old2NewVPValues[&VF] = &NewPlan->VF;
Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
if (BackedgeTakenCount) {
- NewPlan->BackedgeTakenCount = new VPValue();
+ NewPlan->BackedgeTakenCount = new VPSymbolicValue();
Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount;
}
- if (TripCount && TripCount->isLiveIn())
- Old2NewVPValues[TripCount] =
- NewPlan->getOrAddLiveIn(TripCount->getLiveInIRValue());
+ if (auto *LI = dyn_cast_or_null<VPLiveIn>(TripCount))
+ Old2NewVPValues[LI] = NewPlan->getOrAddLiveIn(LI->getValue());
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
@@ -1449,7 +1462,7 @@ void VPSlotTracker::assignName(const VPValue *V) {
const auto &[A, _] = VPValue2Name.try_emplace(V, BaseName);
// Integer or FP constants with different types will result in he same string
// due to stripping types.
- if (V->isLiveIn() && isa<ConstantInt, ConstantFP>(UV))
+ if (isa<VPLiveIn>(V) && isa<ConstantInt, ConstantFP>(UV))
return;
// If it is already used by C > 0 other VPValues, increase the version counter
@@ -1726,10 +1739,10 @@ bool llvm::canConstantBeExtended(const APInt *C, Type *NarrowType,
TargetTransformInfo::OperandValueInfo
VPCostContext::getOperandInfo(VPValue *V) const {
- if (!V->isLiveIn())
- return {};
+ if (auto *LI = dyn_cast<VPLiveIn>(V))
+ return TTI::getOperandInfo(LI->getValue());
- return TTI::getOperandInfo(V->getLiveInIRValue());
+ return {};
}
InstructionCost VPCostContext::getScalarizationOverhead(
@@ -1757,7 +1770,7 @@ InstructionCost VPCostContext::getScalarizationOverhead(
SmallPtrSet<const VPValue *, 4> UniqueOperands;
SmallVector<Type *> Tys;
for (auto *Op : Operands) {
- if (Op->isLiveIn() ||
+ if (isa<VPLiveIn>(Op) ||
(!AlwaysIncludeReplicatingR &&
isa<VPReplicateRecipe, VPPredInstPHIRecipe>(Op)) ||
(isa<VPReplicateRecipe>(Op) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d043ec41ec1ca..6379c4ab21e31 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -528,15 +528,15 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPDefValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPDefValue(this, UV) {}
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPDefID()) {
@@ -1715,9 +1715,9 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
- assert(
- isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
- "last operand must be the called function");
+ assert(isa<Function>(
+ cast<VPLiveIn>(getOperand(getNumOperands() - 1))->getValue()) &&
+ "last operand must be the called function");
}
~VPWidenCallRecipe() override = default;
@@ -1737,7 +1737,8 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
VPCostContext &Ctx) const override;
Function *getCalledScalarFunction() const {
- return cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
+ return cast<Function>(
+ cast<VPLiveIn>(getOperand(getNumOperands() - 1))->getValue());
}
operand_range args() { return drop_end(operands()); }
@@ -2264,7 +2265,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
/// Returns the scalar type of the induction.
Type *getScalarType() const {
return Trunc ? Trunc->getType()
- : getStartValue()->getLiveInIRValue()->getType();
+ : cast<VPLiveIn>(getStartValue())->getValue()->getType();
}
/// Returns the VPValue representing the value of this induction at
@@ -2628,7 +2629,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (Instruction *Inst = IG->getMember(I)) {
if (Inst->getType()->isVoidTy())
continue;
- new VPValue(Inst, this);
+ new VPDefValue(this, Inst);
}
for (auto *SV : StoredValues)
@@ -3132,7 +3133,8 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
assert(Red->getRecurrenceKind() == RecurKind::Add &&
"Expected an add reduction");
assert(getNumOperands() >= 3 && "Expected at least three operands");
- [[maybe_unused]] auto *SubConst = dyn_cast<ConstantInt>(getOperand(2)->getLiveInIRValue());
+ [[maybe_unused]] auto *SubConst =
+ dyn_cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue());
assert(SubConst && SubConst->getValue() == 0 &&
Sub->getOpcode() == Instruction::Sub && "Expected a negating sub");
}
@@ -3350,13 +3352,13 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPValue {
+ public VPDefValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, bool Reverse,
const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
Reverse, Metadata, DL),
- VPValue(this, &Load) {
+ VPDefValue(this, &Load) {
setMask(Mask);
}
@@ -3391,13 +3393,14 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// A recipe for widening load operations with vector-predication intrinsics,
/// using the address to load from, the explicit vector length and an optional
/// mask.
-struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
+struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
+ public VPDefValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
L.getDebugLoc()),
- VPValue(this, &getIngredient()) {
+ VPDefValue(this, &getIngredient()) {
setMask(Mask);
}
@@ -3582,7 +3585,7 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
/// Returns the scalar type of the induction.
Type *getScalarType() const {
- return getStartValue()->getLiveInIRValue()->getType();
+ return cast<VPLiveIn>(getStartValue())->getValue()->getType();
}
/// Returns true if the recipe only uses the first lane of operand \p Op.
@@ -3773,7 +3776,7 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
}
Type *getScalarType() const {
- return getStartValue()->getLiveInIRValue()->getType();
+ return cast<VPLiveIn>(getStartValue())->getValue()->getType();
}
VPValue *getStartValue() const { return getOperand(0); }
@@ -4330,20 +4333,20 @@ class VPlan {
/// Represents the backedge taken count of the original loop, for folding
/// the tail. It equals TripCount - 1.
- VPValue *BackedgeTakenCount = nullptr;
+ VPSymbolicValue *BackedgeTakenCount = nullptr;
/// Represents the vector trip count.
- VPValue VectorTripCount;
+ VPSymbolicValue VectorTripCount;
/// Represents the vectorization factor of the loop.
- VPValue VF;
+ VPSymbolicValue VF;
/// Represents the loop-invariant VF * UF of the vector loop region.
- VPValue VFxUF;
+ VPSymbolicValue VFxUF;
/// Contains all the external definitions created for this VPlan, as a mapping
- /// from IR Values to VPValues.
- SmallMapVector<Value *, VPValue *, 16> LiveIns;
+ /// from IR Values to VPLiveIns.
+ SmallMapVector<Value *, VPLiveIn *, 16> LiveIns;
/// Blocks allocated and owned by the VPlan. They will be deleted once the
/// VPlan is destroyed.
@@ -4469,7 +4472,7 @@ class VPlan {
/// The backedge taken count of the original loop.
VPValue *getOrCreateBackedgeTakenCount() {
if (!BackedgeTakenCount)
- BackedgeTakenCount = new VPValue();
+ BackedgeTakenCount = new VPSymbolicValue();
return BackedgeTakenCount;
}
VPValue *getBackedgeTakenCount() const { return BackedgeTakenCount; }
@@ -4538,43 +4541,42 @@ class VPlan {
/// Gets the live-in VPValue for \p V or adds a new live-in (if none exists
/// yet) for \p V.
- VPValue *getOrAddLiveIn(Value *V) {
+ VPLiveIn *getOrAddLiveIn(Value *V) {
assert(V && "Trying to get or add the VPValue of a null Value");
auto [It, Inserted] = LiveIns.try_emplace(V);
- if (Inserted) {
- VPValue *VPV = new VPValue(V);
- assert(VPV->isLiveIn() && "VPV must be a live-in.");
- It->second = VPV;
- }
+ if (Inserted)
+ It->second = new VPLiveIn(V);
- assert(It->second->isLiveIn() && "Only live-ins should be in mapping");
+ assert(isa<VPLiveIn>(It->second) && "Only VPLiveIns should be in mapping");
return It->second;
}
- /// Return a VPValue wrapping i1 true.
- VPValue *getTrue() { return getConstantInt(1, 1); }
+ /// Return a VPLiveIn wrapping i1 true.
+ VPLiveIn *getTrue() { return getConstantInt(1, 1); }
- /// Return a VPValue wrapping i1 false.
- VPValue *getFalse() { return getConstantInt(1, 0); }
+ /// Return a VPLiveIn wrapping i1 false.
+ VPLiveIn *getFalse() { return getConstantInt(1, 0); }
- /// Return a VPValue wrapping a ConstantInt with the given type and value.
- VPValue *getConstantInt(Type *Ty, uint64_t Val, bool IsSigned = false) {
+ /// Return a VPLiveIn wrapping a ConstantInt with the given type and value.
+ VPLiveIn *getConstantInt(Type *Ty, uint64_t Val, bool IsSigned = false) {
return getOrAddLiveIn(ConstantInt::get(Ty, Val, IsSigned));
}
- /// Return a VPValue wrapping a ConstantInt with the given bitwidth and value.
- VPValue *getConstantInt(unsigned BitWidth, uint64_t Val,
- bool IsSigned = false) {
+ /// Return a VPLiveIn wrapping a ConstantInt with the given bitwidth and
+ /// value.
+ VPLiveIn *getConstantInt(unsigned BitWidth, uint64_t Val,
+ bool IsSigned = false) {
return getConstantInt(APInt(BitWidth, Val, IsSigned));
}
- /// Return a VPValue wrapping a ConstantInt with the given APInt value.
- VPValue *getConstantInt(const APInt &Val) {
+ /// Return a VPLiveIn wrapping a ConstantInt with the given APInt value.
+ VPLiveIn *getConstantInt(const APInt &Val) {
return getOrAddLiveIn(ConstantInt::get(getContext(), Val));
}
- /// Return the live-in VPValue for \p V, if there is one or nullptr otherwise.
- VPValue *getLiveIn(Value *V) const { return LiveIns.lookup(V); }
+ /// Return the live-in VPLiveIn for \p V, if there is one or nullptr
+ /// otherwise.
+ VPLiveIn *getLiveIn(Value *V) const { return LiveIns.lookup(V); }
/// Return the list of live-in VPValues available in the VPlan.
auto getLiveIns() const { return LiveIns.values(); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index a586aafa2855d..bd1edc85926f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -36,8 +36,8 @@ VPTypeAnalysis::VPTypeAnalysis(const VPlan &Plan) : Ctx(Plan.getContext()) {
// If there's no canonical IV, retrieve the type from the trip count
// expression.
auto *TC = Plan.getTripCount();
- if (TC->isLiveIn()) {
- CanonicalIVTy = TC->getLiveInIRValue()->getType();
+ if (auto *TCLI = dyn_cast<VPLiveIn>(TC)) {
+ CanonicalIVTy = TCLI->getType();
return;
}
CanonicalIVTy = cast<VPExpandSCEVRecipe>(TC)->getSCEV()->getType();
@@ -169,7 +169,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenRecipe *R) {
case Instruction::ExtractValue: {
assert(R->getNumOperands() == 2 && "expected single level extractvalue");
auto *StructTy = cast<StructType>(inferScalarType(R->getOperand(0)));
- auto *CI = cast<ConstantInt>(R->getOperand(1)->getLiveInIRValue());
+ auto *CI = cast<ConstantInt>(cast<VPLiveIn>(R->getOperand(1))->getValue());
return StructTy->getTypeAtIndex(CI->getZExtValue());
}
default:
@@ -222,7 +222,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
switch (Opcode) {
case Instruction::Call: {
unsigned CallIdx = R->getNumOperands() - (R->isPredicated() ? 2 : 1);
- return cast<Function>(R->getOperand(CallIdx)->getLiveInIRValue())
+ return cast<Function>(cast<VPLiveIn>(R->getOperand(CallIdx))->getValue())
->getReturnType();
}
case Instruction::Select: {
@@ -264,9 +264,10 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
- if (V->isLiveIn()) {
- if (auto *IRValue = V->getLiveInIRValue())
- return IRValue->getType();
+ if (auto *LI = dyn_cast<VPLiveIn>(V))
+ return LI->getType();
+
+ if (isa<VPSymbolicValue>(V)) {
// All VPValues without any underlying IR value (like the vector trip count
// or the backedge-taken count) have the same type as the canonical IV.
return CanonicalIVTy;
@@ -447,8 +448,8 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
// FIXME: Might need some motivation why these values are ignored. If
// for example an argument is used inside the loop it will increase the
// register pressure (so shouldn't we add it to LoopInvariants).
- if (!DefR && (!U->getLiveInIRValue() ||
- !isa<Instruction>(U->getLiveInIRValue())))
+ auto *LI = dyn_cast<VPLiveIn>(U);
+ if (!DefR && (!LI || !isa<Instruction>(LI->getValue())))
continue;
// If this recipe is outside the loop then record it and continue.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index f082b970c7762..ec5f8ac4d009e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -106,13 +106,11 @@ template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
int_pred_ty() : P() {}
bool match(VPValue *VPV) const {
- if (!VPV->isLiveIn())
+ auto *LI = dyn_cast<VPLiveIn>(VPV);
+ if (!LI)
return false;
- Value *V = VPV->getLiveInIRValue();
- if (!V)
- return false;
- assert(!V->getType()->isVectorTy() && "Unexpected vector live-in");
- const auto *CI = dyn_cast<ConstantInt>(V);
+ assert(!LI->getType()->isVectorTy() && "Unexpected vector live-in");
+ const auto *CI = dyn_cast<ConstantInt>(LI->getValue());
if (!CI)
return false;
@@ -182,13 +180,11 @@ struct bind_apint {
bind_apint(const APInt *&Res) : Res(Res) {}
bool match(VPValue *VPV) const {
- if (!VPV->isLiveIn())
- return false;
- Value *V = VPV->getLiveInIRValue();
- if (!V)
+ auto *LI = dyn_cast<VPLiveIn>(VPV);
+ if (!LI)
return false;
- assert(!V->getType()->isVectorTy() && "Unexpected vector live-in");
- const auto *CI = dyn_cast<ConstantInt>(V);
+ assert(!LI->getType()->isVectorTy() && "Unexpected vector live-in");
+ const auto *CI = dyn_cast<ConstantInt>(LI->getValue());
if (!CI)
return false;
Res = &CI->getValue();
@@ -901,8 +897,7 @@ m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
struct live_in_vpvalue {
template <typename ITy> bool match(ITy *V) const {
- VPValue *Val = dyn_cast<VPValue>(V);
- return Val && Val->isLiveIn();
+ return isa<VPLiveIn, VPSymbolicValue>(V);
}
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ddbf014c17d4f..8631d2fad2f68 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -394,7 +394,8 @@ VPUnrollPartAccessor<PartOpIdx>::getUnrollPartOperand(const VPUser &U) const {
template <unsigned PartOpIdx>
unsigned VPUnrollPartAccessor<PartOpIdx>::getUnrollPart(const VPUser &U) const {
if (auto *UnrollPartOp = getUnrollPartOperand(U))
- return cast<ConstantInt>(UnrollPartOp->getLiveInIRValue())->getZExtValue();
+ return cast<ConstantInt>(cast<VPLiveIn>(UnrollPartOp)->getValue())
+ ->getZExtValue();
return 0;
}
@@ -532,9 +533,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case Instruction::ExtractElement: {
assert(State.VF.isVector() && "Only extract elements from vectors");
- if (getOperand(1)->isLiveIn()) {
+ if (auto *IdxLI = dyn_cast<VPLiveIn>(getOperand(1))) {
unsigned IdxToExtract =
- cast<ConstantInt>(getOperand(1)->getLiveInIRValue())->getZExtValue();
+ cast<ConstantInt>(IdxLI->getValue())->getZExtValue();
return State.get(getOperand(0), VPLane(IdxToExtract));
}
Value *Vec = State.get(getOperand(0));
@@ -577,7 +578,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
Name);
ElementCount EC = State.VF.multiplyCoefficientBy(
- cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue());
+ cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue())
+ ->getZExtValue());
auto *PredTy = VectorType::get(Builder.getInt1Ty(), EC);
return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
{PredTy, ScalarTC->getType()},
@@ -694,7 +696,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
// If this start vector is scaled then it should produce a vector with fewer
// elements than the VF.
ElementCount VF = State.VF.divideCoefficientBy(
- cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue());
+ cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue())
+ ->getZExtValue());
auto *Iden = Builder.CreateVectorSplat(VF, State.get(getOperand(1), true));
return Builder.CreateInsertElement(Iden, State.get(getOperand(0), true),
Builder.getInt32(0));
@@ -738,7 +741,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
State.get(getOperand(3 + Part)));
Value *Start = State.get(getOperand(1), true);
- Value *Sentinel = getOperand(2)->getLiveInIRValue();
+ Value *Sentinel = cast<VPLiveIn>(getOperand(2))->getValue();
return createFindLastIVReduction(Builder, ReducedPartRdx, RK, Start,
Sentinel);
}
@@ -1073,7 +1076,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask: {
Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));
unsigned Multiplier =
- cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue();
+ cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue())
+ ->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
{ArgTy, ArgTy});
@@ -1487,7 +1491,7 @@ void VPIRInstruction::extractLastLaneOfLastPartOfFirstOperand(
"can only update exiting operands to phi nodes");
assert(getNumOperands() > 0 && "must have at least one operand");
VPValue *Exiting = getOperand(0);
- if (Exiting->isLiveIn())
+ if (isa<VPLiveIn>(Exiting))
return;
Exiting = Builder.createNaryOp(VPInstruction::ExtractLastPart, Exiting);
@@ -1849,8 +1853,8 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// a multiply, and add that into the cost.
InstructionCost MulCost =
Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (IncAmt->isLiveIn()) {
- ConstantInt *CI = dyn_cast<ConstantInt>(IncAmt->getLiveInIRValue());
+ if (auto *IncAmtLI = dyn_cast<VPLiveIn>(IncAmt)) {
+ ConstantInt *CI = dyn_cast<ConstantInt>(IncAmtLI->getValue());
if (CI && CI->getZExtValue() == 1)
MulCost = TTI::TCC_Free;
@@ -2105,7 +2109,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
case Instruction::ExtractValue: {
assert(getNumOperands() == 2 && "expected single level extractvalue");
Value *Op = State.get(getOperand(0));
- auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
+ auto *CI = cast<ConstantInt>(cast<VPLiveIn>(getOperand(1))->getValue());
Value *Extract = Builder.CreateExtractValue(Op, CI->getZExtValue());
State.set(this, Extract);
break;
@@ -2252,7 +2256,7 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,
// For Z/Sext, get the context from the operand.
else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
Opcode == Instruction::FPExt) {
- if (Operand->isLiveIn())
+ if (isa<VPLiveIn>(Operand))
CCH = TTI::CastContextHint::Normal;
else if (Operand->getDefiningRecipe())
CCH = ComputeCCH(Operand->getDefiningRecipe());
@@ -2311,8 +2315,10 @@ bool VPWidenIntOrFpInductionRecipe::isCanonical() const {
// 1, which is represented as live-in.
if (getStepValue()->getDefiningRecipe())
return false;
- auto *StepC = dyn_cast<ConstantInt>(getStepValue()->getLiveInIRValue());
- auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
+ auto *StepC =
+ dyn_cast<ConstantInt>(cast<VPLiveIn>(getStepValue())->getValue());
+ auto *StartC =
+ dyn_cast<ConstantInt>(cast<VPLiveIn>(getStartValue())->getValue());
return StartC && StartC->isZero() && StepC && StepC->isOne() &&
getScalarType() == getRegion()->getCanonicalIVType();
}
@@ -2751,7 +2757,7 @@ VPExpressionRecipe::VPExpressionRecipe(
if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))
continue;
addOperand(Op);
- LiveInPlaceholders.push_back(new VPValue());
+ LiveInPlaceholders.push_back(new VPSymbolicValue());
}
}
@@ -3183,8 +3189,8 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
// instruction cost.
return 0;
case Instruction::Call: {
- auto *CalledFn =
- cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
+ auto *CalledFn = cast<Function>(
+ cast<VPLiveIn>(getOperand(getNumOperands() - 1))->getValue());
SmallVector<const VPValue *> ArgOps(drop_end(operands()));
SmallVector<Type *, 4> Tys;
@@ -3890,7 +3896,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// TODO: Also manage existing metadata using VPIRMetadata.
Group->addMetadata(NewLoad);
- ArrayRef<VPValue *> VPDefs = definedValues();
+ ArrayRef<VPDefValue *> VPDefs = definedValues();
if (VecTy->isScalableTy()) {
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
@@ -4287,7 +4293,7 @@ void VPWidenCanonicalIVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
// Create a vector from the initial value.
- auto *VectorInit = getStartValue()->getLiveInIRValue();
+ auto *VectorInit = cast<VPLiveIn>(getStartValue())->getValue();
Type *VecTy = State.VF.isScalar()
? VectorInit->getType()
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 25a4b60e9a533..9e89c06b4fc75 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1151,9 +1151,12 @@ static VPValue *tryToFoldLiveIns(VPSingleDefRecipe &R,
SmallVector<Value *, 4> Ops;
for (VPValue *Op : Operands) {
- if (!Op->isLiveIn() || !Op->getLiveInIRValue())
+ if (!isa<VPLiveIn, VPSymbolicValue>(Op))
return nullptr;
- Ops.push_back(Op->getLiveInIRValue());
+ Value *V = Op->getUnderlyingValue();
+ if (!V)
+ return nullptr;
+ Ops.push_back(V);
}
auto FoldToIRValue = [&]() -> Value * {
@@ -1224,7 +1227,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
// Fold PredPHI LiveIn -> LiveIn.
if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Def)) {
VPValue *Op = PredPHI->getOperand(0);
- if (Op->isLiveIn())
+ if (isa<VPLiveIn>(Op))
PredPHI->replaceAllUsesWith(Op);
}
@@ -1504,7 +1507,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
return;
// Hoist an invariant increment Y of a phi X, by having X start at Y.
- if (match(Def, m_c_Add(m_VPValue(X), m_VPValue(Y))) && Y->isLiveIn() &&
+ if (match(Def, m_c_Add(m_VPValue(X), m_VPValue(Y))) && isa<VPLiveIn>(Y) &&
isa<VPPhi>(X)) {
auto *Phi = cast<VPPhi>(X);
if (Phi->getOperand(1) != Def && match(Phi->getOperand(0), m_ZeroInt()) &&
@@ -1664,8 +1667,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
return false;
// Non-constant live-ins require broadcasts, while constants do not
// need explicit broadcasts.
- bool LiveInNeedsBroadcast =
- Op->isLiveIn() && !isa<Constant>(Op->getLiveInIRValue());
+ auto *LI = dyn_cast<VPLiveIn>(Op);
+ bool LiveInNeedsBroadcast = LI && !isa<Constant>(LI->getValue());
auto *OpR = dyn_cast<VPReplicateRecipe>(Op);
return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
}))
@@ -2577,7 +2580,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
}
VPBuilder Builder;
- if (Op->isLiveIn())
+ if (isa<VPLiveIn>(Op))
Builder.setInsertPoint(PH);
else
Builder.setInsertPoint(&R);
@@ -3874,7 +3877,7 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
}
VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx);
- if (!IncomingFromEarlyExit->isLiveIn()) {
+ if (!isa<VPLiveIn>(IncomingFromEarlyExit)) {
// Update the incoming value from the early exit.
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {CondToEarlyExit},
@@ -4044,7 +4047,7 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
auto ExtendAndReplaceConstantOp = [&Ctx](VPWidenCastRecipe *ExtA,
VPWidenCastRecipe *&ExtB,
VPValue *&ValB, VPWidenRecipe *Mul) {
- if (!ExtA || ExtB || !ValB->isLiveIn())
+ if (!ExtA || ExtB || !isa<VPLiveIn>(ValB))
return;
Type *NarrowTy = Ctx.Types.inferScalarType(ExtA->getOperand(0));
Instruction::CastOps ExtOpc = ExtA->getOpcode();
@@ -4184,8 +4187,7 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
auto *VectorPreheader = Plan.getVectorPreheader();
for (VPValue *VPV : VPValues) {
if (vputils::onlyScalarValuesUsed(VPV) ||
- (VPV->isLiveIn() && VPV->getLiveInIRValue() &&
- isa<Constant>(VPV->getLiveInIRValue())))
+ (isa<VPLiveIn>(VPV) && isa<Constant>(cast<VPLiveIn>(VPV)->getValue())))
continue;
// Add explicit broadcast at the insert point that dominates all users.
@@ -4488,7 +4490,7 @@ void VPlanTransforms::materializeConstantVectorTripCount(
if (!Plan.hasScalarTail() ||
Plan.getMiddleBlock()->getSingleSuccessor() ==
Plan.getScalarPreheader() ||
- !TC->isLiveIn())
+ !isa<VPLiveIn>(TC))
return;
// Materialize vector trip counts for constants early if it can simply
@@ -4496,7 +4498,7 @@ void VPlanTransforms::materializeConstantVectorTripCount(
// TODO: Compute vector trip counts for loops requiring a scalar epilogue and
// tail-folded loops.
ScalarEvolution &SE = *PSE.getSE();
- auto *TCScev = SE.getSCEV(TC->getLiveInIRValue());
+ auto *TCScev = SE.getSCEV(cast<VPLiveIn>(TC)->getValue());
if (!isa<SCEVConstant>(TCScev))
return;
const SCEV *VFxUF = SE.getElementCount(TCScev->getType(), BestVF * BestUF);
@@ -4617,10 +4619,11 @@ void VPlanTransforms::materializeVectorTripCount(VPlan &Plan,
bool TailByMasking,
bool RequiresScalarEpilogue) {
VPValue &VectorTC = Plan.getVectorTripCount();
- assert(VectorTC.isLiveIn() && "vector-trip-count must be a live-in");
+ assert(isa<VPSymbolicValue>(VectorTC) &&
+ "vector-trip-count must be a live-in");
// There's nothing to do if there are no users of the vector trip count or its
// IR value has already been set.
- if (VectorTC.getNumUsers() == 0 || VectorTC.getLiveInIRValue())
+ if (VectorTC.getNumUsers() == 0 || VectorTC.getUnderlyingValue())
return;
VPValue *TC = Plan.getTripCount();
@@ -4807,7 +4810,7 @@ static bool isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
/// Returns true if \p VPValue is a narrow VPValue.
static bool isAlreadyNarrow(VPValue *VPV) {
- if (VPV->isLiveIn())
+ if (isa<VPLiveIn>(VPV))
return true;
auto *RepR = dyn_cast<VPReplicateRecipe>(VPV);
return RepR && RepR->isSingleScalar();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 8198945764936..7d83d77cb57e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -79,7 +79,7 @@ class UnrollState {
void unrollBlock(VPBlockBase *VPB);
VPValue *getValueForPart(VPValue *V, unsigned Part) {
- if (Part == 0 || V->isLiveIn())
+ if (Part == 0 || isa<VPLiveIn, VPSymbolicValue>(V))
return V;
assert((VPV2Parts.contains(V) && VPV2Parts[V].size() >= Part) &&
"accessed value does not exist");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 25052b7c8b1b3..fe1fe4bc0dcdc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -81,8 +81,8 @@ bool vputils::isHeaderMask(const VPValue *V, const VPlan &Plan) {
const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
ScalarEvolution &SE, const Loop *L) {
- if (V->isLiveIn()) {
- Value *LiveIn = V->getLiveInIRValue();
+ if (isa<VPLiveIn, VPSymbolicValue>(V)) {
+ Value *LiveIn = V->getUnderlyingValue();
if (LiveIn && SE.isSCEVable(LiveIn->getType()))
return SE.getSCEV(LiveIn);
return SE.getCouldNotCompute();
@@ -172,7 +172,7 @@ static bool preservesUniformity(unsigned Opcode) {
bool vputils::isSingleScalar(const VPValue *VPV) {
// A live-in must be uniform across the scope of VPlan.
- if (VPV->isLiveIn())
+ if (isa<VPLiveIn, VPSymbolicValue>(VPV))
return true;
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
@@ -210,7 +210,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
// Live-ins are uniform.
- if (V->isLiveIn())
+ if (isa<VPLiveIn, VPSymbolicValue>(V))
return true;
VPRecipeBase *R = V->getDefiningRecipe();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8849ccc6184fc..4fb15af08037d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -30,47 +30,35 @@ namespace llvm {
// Forward declarations.
class raw_ostream;
+class Type;
class Value;
class VPDef;
struct VPDoubleValueDef;
class VPSlotTracker;
class VPUser;
class VPRecipeBase;
-class VPInterleaveBase;
class VPPhiAccessors;
/// This is the base class of the VPlan Def/Use graph, used for modeling the
/// data flow into, within and out of the VPlan. VPValues can stand for live-ins
-/// coming from the input IR and instructions which VPlan will generate if
-/// executed.
+/// coming from the input IR, symbolic values and values defined by recipes.
class LLVM_ABI_FOR_TEST VPValue {
friend class VPDef;
friend struct VPDoubleValueDef;
- friend class VPInterleaveBase;
friend class VPlan;
- friend class VPExpressionRecipe;
+ friend struct VPLiveIn;
+ friend struct VPSymbolicValue;
+ friend class VPDefValue;
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
SmallVector<VPUser *, 1> Users;
-protected:
/// Hold the underlying Value, if any, attached to this VPValue.
Value *UnderlyingVal;
- /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the
- /// VPValue is not defined by any recipe modeled in VPlan.
- VPDef *Def;
-
VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
- /// Create a live-in VPValue.
- VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {}
- /// Create a VPValue for a \p Def which is a subclass of VPValue.
- VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {}
- /// Create a VPValue for a \p Def which defines multiple values.
- VPValue(Value *UV, VPDef *Def) : VPValue(VPValueSC, UV, Def) {}
-
// DESIGN PRINCIPLE: Access to the underlying IR must be strictly limited to
// the front-end and back-end of VPlan so that the middle-end is as
// independent as possible of the underlying IR. We grant access to the
@@ -85,9 +73,9 @@ class LLVM_ABI_FOR_TEST VPValue {
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
- VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe
- /// that defines multiple values.
- VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase.
+ VPVLiveInSC, /// A live-in VPValue wrapping an IR Value.
+ VPVSymbolicSC, /// A symbolic live-in VPValue without IR backing.
+ VPVDefValueSC, /// A VPValue defined by a recipe.
};
VPValue(const VPValue &) = delete;
@@ -172,18 +160,6 @@ class LLVM_ABI_FOR_TEST VPValue {
/// Returns true if this VPValue is defined by a recipe.
bool hasDefiningRecipe() const { return getDefiningRecipe(); }
- /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
- bool isLiveIn() const { return !hasDefiningRecipe(); }
-
- /// Returns the underlying IR value, if this VPValue is defined outside the
- /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef
- /// inside a VPlan.
- Value *getLiveInIRValue() const {
- assert(isLiveIn() &&
- "VPValue is not a live-in; it is defined by a VPDef inside a VPlan");
- return getUnderlyingValue();
- }
-
/// Returns true if the VPValue is defined outside any loop.
bool isDefinedOutsideLoopRegions() const;
@@ -197,6 +173,52 @@ class LLVM_ABI_FOR_TEST VPValue {
LLVM_ABI_FOR_TEST raw_ostream &operator<<(raw_ostream &OS,
const VPRecipeBase &R);
+/// A VPValue representing a live-in from the input IR. It wraps an underlying
+/// IR Value.
+struct VPLiveIn : public VPValue {
+ VPLiveIn(Value *UV) : VPValue(VPVLiveInSC, UV, nullptr) {
+ assert(UV && "VPLiveIn requires an underlying IR value");
+ }
+
+ /// Returns the underlying IR value.
+ Value *getValue() const { return getUnderlyingValue(); }
+
+ /// Returns the type of the underlying IR value.
+ Type *getType() const;
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVLiveInSC;
+ }
+};
+
+/// A symbolic live-in VPValue, used for values like vector trip count, VF, and
+/// VFxUF.
+struct VPSymbolicValue : public VPValue {
+ VPSymbolicValue() : VPValue(VPVSymbolicSC, nullptr, nullptr) {}
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSymbolicSC;
+ }
+};
+
+/// A VPValue defined by a recipe that produces multiple values.
+class VPDefValue : public VPValue {
+ friend class VPValue;
+ friend class VPDef;
+ /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the
+ /// VPValue is not defined by any recipe modeled in VPlan.
+ VPDef *Def;
+
+public:
+ VPDefValue(VPDef *Def, Value *UV = nullptr);
+
+ virtual ~VPDefValue();
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVDefValueSC;
+ }
+};
+
/// This class augments VPValue with operands which provide the inverse def-use
/// edges from VPValue's users to their defs.
class VPUser {
@@ -304,15 +326,16 @@ class VPUser {
/// from VPDef before VPValue.
class VPDef {
friend class VPValue;
+ friend class VPDefValue;
/// Subclass identifier (for isa/dyn_cast).
const unsigned char SubclassID;
/// The VPValues defined by this VPDef.
- TinyPtrVector<VPValue *> DefinedValues;
+ TinyPtrVector<VPDefValue *> DefinedValues;
/// Add \p V as a defined value by this VPDef.
- void addDefinedValue(VPValue *V) {
+ void addDefinedValue(VPDefValue *V) {
assert(V->Def == this &&
"can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
@@ -320,7 +343,7 @@ class VPDef {
/// Remove \p V from the values defined by this VPDef. \p V must be a defined
/// value of this VPDef.
- void removeDefinedValue(VPValue *V) {
+ void removeDefinedValue(VPDefValue *V) {
assert(V->Def == this && "can only remove VPValue linked with this VPDef");
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
@@ -384,7 +407,7 @@ class VPDef {
VPDef(const unsigned char SC) : SubclassID(SC) {}
virtual ~VPDef() {
- for (VPValue *D : make_early_inc_range(DefinedValues)) {
+ for (VPDefValue *D : make_early_inc_range(DefinedValues)) {
assert(D->Def == this &&
"all defined VPValues should point to the containing VPDef");
assert(D->getNumUsers() == 0 &&
@@ -418,9 +441,9 @@ class VPDef {
}
/// Returns an ArrayRef of the values defined by the VPDef.
- ArrayRef<VPValue *> definedValues() { return DefinedValues; }
+ ArrayRef<VPDefValue *> definedValues() { return DefinedValues; }
/// Returns an ArrayRef of the values defined by the VPDef.
- ArrayRef<VPValue *> definedValues() const { return DefinedValues; }
+ ArrayRef<VPDefValue *> definedValues() const { return DefinedValues; }
/// Returns the number of values defined by the VPDef.
unsigned getNumDefinedValues() const { return DefinedValues.size(); }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a6a1f672c5f70..517e166a78656 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1709,8 +1709,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPValue(nullptr, this);
- new VPValue(nullptr, this);
+ new VPDefValue(this);
+ new VPDefValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
>From 2d112203482b9ce2c6d3b866ef6990d239ab478c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 28 Dec 2025 13:45:34 +0000
Subject: [PATCH 2/4] !fixup addres socmments, thanks!
---
.../Vectorize/LoopVectorizationPlanner.h | 2 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 6 ++--
llvm/lib/Transforms/Vectorize/VPlan.cpp | 17 +++++-----
llvm/lib/Transforms/Vectorize/VPlan.h | 31 ++++++++++---------
.../Vectorize/VPlanConstruction.cpp | 4 +--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 10 +++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +++----
llvm/lib/Transforms/Vectorize/VPlanValue.h | 5 ++-
8 files changed, 44 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 430d9469c7698..eb6ca43aec490 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -296,7 +296,7 @@ class VPBuilder {
/// induction with \p Start and \p Step values, using \p Start + \p Current *
/// \p Step.
VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
- FPMathOperator *FPBinOp, VPValue *Start,
+ FPMathOperator *FPBinOp, VPLiveIn *Start,
VPValue *Current, VPValue *Step,
const Twine &Name = "") {
return tryInsertInstruction(
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 81ecd6e669701..e493b7861e2d6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7426,11 +7426,11 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
DenseMap<const SCEV *, Value *> ExpandedSCEVs =
VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
if (!ILV.getTripCount()) {
- // After expandSCEVs, TripCount is always a VPLiveIn.
ILV.setTripCount(cast<VPLiveIn>(BestVPlan.getTripCount())->getValue());
- } else
+ } else {
assert(VectorizingEpilogue && "should only re-use the existing trip "
"count during epilogue vectorization");
+ }
// Perform the actual loop transformation.
VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
@@ -7749,7 +7749,7 @@ VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
auto *WidenIV = cast<VPWidenIntOrFpInductionRecipe>(
VPI->getOperand(0)->getDefiningRecipe());
PHINode *Phi = WidenIV->getPHINode();
- VPValue *Start = WidenIV->getStartValue();
+ VPLiveIn *Start = WidenIV->getStartValue();
const InductionDescriptor &IndDesc = WidenIV->getInductionDescriptor();
// It is always safe to copy over the NoWrap and FastMath flags. In
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 7d6c694d10db9..885dcb92d8a74 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -78,8 +78,6 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const VPRecipeBase &R) {
}
#endif
-Type *VPLiveIn::getType() const { return getUnderlyingValue()->getType(); }
-
Value *VPLane::getAsRuntimeExpr(IRBuilderBase &Builder,
const ElementCount &VF) const {
switch (LaneKind) {
@@ -126,19 +124,22 @@ void VPDef::dump() const {
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
- auto *Def = dyn_cast<VPDefValue>(this);
- if (!Def)
+ auto *DefValue = dyn_cast<VPDefValue>(this);
+ if (!DefValue)
return nullptr;
- return cast<VPRecipeBase>(Def->Def);
+ return cast<VPRecipeBase>(DefValue->Def);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *Def = dyn_cast<VPDefValue>(this);
- if (!Def)
+ auto *DefValue = dyn_cast<VPDefValue>(this);
+ if (!DefValue)
return nullptr;
- return cast<VPRecipeBase>(Def->Def);
+ assert(DefValue);
+ return cast<VPRecipeBase>(DefValue->Def);
}
+Type *VPLiveIn::getType() const { return getUnderlyingValue()->getType(); }
+
VPDefValue::VPDefValue(VPDef *Def, Value *UV)
: VPValue(VPVDefValueSC, UV, nullptr), Def(Def) {
assert(Def && "VPDefValue requires a defining recipe");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b93e41abdf6db..d26ebd6f32718 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2148,6 +2148,8 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
void execute(VPTransformState &State) override = 0;
+ VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+
/// Returns the step value of the induction.
VPValue *getStepValue() { return getOperand(1); }
const VPValue *getStepValue() const { return getOperand(1); }
@@ -2204,7 +2206,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
bool isUnrolled() const { return getNumOperands() == 5; }
public:
- VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
+ VPWidenIntOrFpInductionRecipe(PHINode *IV, VPLiveIn *Start, VPValue *Step,
VPValue *VF, const InductionDescriptor &IndDesc,
const VPIRFlags &Flags, DebugLoc DL)
: VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start,
@@ -2213,7 +2215,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
addOperand(VF);
}
- VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
+ VPWidenIntOrFpInductionRecipe(PHINode *IV, VPLiveIn *Start, VPValue *Step,
VPValue *VF, const InductionDescriptor &IndDesc,
TruncInst *Trunc, const VPIRFlags &Flags,
DebugLoc DL)
@@ -2243,6 +2245,8 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
"expandVPWidenIntOrFpInductionRecipe");
}
+ VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+
VPValue *getSplatVFValue() {
// If the recipe has been unrolled return the VPValue for the induction
// increment.
@@ -2266,8 +2270,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
/// Returns the scalar type of the induction.
Type *getScalarType() const {
- return Trunc ? Trunc->getType()
- : cast<VPLiveIn>(getStartValue())->getValue()->getType();
+ return Trunc ? Trunc->getType() : getStartValue()->getType();
}
/// Returns the VPValue representing the value of this induction at
@@ -3567,13 +3570,13 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
/// canonical induction variable.
class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
public:
- VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
+ VPCanonicalIVPHIRecipe(VPLiveIn *StartV, DebugLoc DL)
: VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV, DL) {}
~VPCanonicalIVPHIRecipe() override = default;
VPCanonicalIVPHIRecipe *clone() override {
- auto *R = new VPCanonicalIVPHIRecipe(getOperand(0), getDebugLoc());
+ auto *R = new VPCanonicalIVPHIRecipe(getStartValue(), getDebugLoc());
R->addOperand(getBackedgeValue());
return R;
}
@@ -3585,10 +3588,10 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
"scalar phi recipe");
}
+ VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+
/// Returns the scalar type of the induction.
- Type *getScalarType() const {
- return cast<VPLiveIn>(getStartValue())->getValue()->getType();
- }
+ Type *getScalarType() const { return getStartValue()->getType(); }
/// Returns true if the recipe only uses the first lane of operand \p Op.
bool usesFirstLaneOnly(const VPValue *Op) const override {
@@ -3743,7 +3746,7 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
std::string Name;
public:
- VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start,
+ VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPLiveIn *Start,
VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
const Twine &Name = "")
: VPDerivedIVRecipe(
@@ -3752,7 +3755,7 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
Start, CanonicalIV, Step, Name) {}
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
- const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV,
+ const FPMathOperator *FPBinOp, VPLiveIn *Start, VPValue *IV,
VPValue *Step, const Twine &Name = "")
: VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, IV, Step}), Kind(Kind),
FPBinOp(FPBinOp), Name(Name.str()) {}
@@ -3777,11 +3780,9 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
return 0;
}
- Type *getScalarType() const {
- return cast<VPLiveIn>(getStartValue())->getValue()->getType();
- }
+ Type *getScalarType() const { return getStartValue()->getType(); }
- VPValue *getStartValue() const { return getOperand(0); }
+ VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
VPValue *getStepValue() const { return getOperand(2); }
/// Returns true if the recipe only uses the first lane of operand \p Op.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a087322fe862f..4ae1fc239d9be 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -599,7 +599,7 @@ VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
/// Creates a VPWidenIntOrFpInductionRecipe or VPWidenPointerInductionRecipe
/// for \p Phi based on \p IndDesc.
static VPHeaderPHIRecipe *
-createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPValue *Start,
+createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPLiveIn *Start,
const InductionDescriptor &IndDesc, VPlan &Plan,
PredicatedScalarEvolution &PSE, Loop &OrigLoop,
DebugLoc DL) {
@@ -660,7 +660,7 @@ void VPlanTransforms::createHeaderPhiRecipes(
"Must have 2 operands for header phis");
// Extract common values once.
- VPValue *Start = PhiR->getOperand(0);
+ VPLiveIn *Start = cast<VPLiveIn>(PhiR->getOperand(0));
VPValue *BackedgeValue = PhiR->getOperand(1);
if (FixedOrderRecurrences.contains(Phi)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d801928a7d974..19c89168b37a1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2348,12 +2348,12 @@ bool VPWidenIntOrFpInductionRecipe::isCanonical() const {
// The step may be defined by a recipe in the preheader (e.g. if it requires
// SCEV expansion), but for the canonical induction the step is required to be
// 1, which is represented as live-in.
- if (getStepValue()->getDefiningRecipe())
+ const VPLiveIn *Step = dyn_cast<VPLiveIn>(getStepValue());
+ if (!Step)
return false;
- auto *StepC =
- dyn_cast<ConstantInt>(cast<VPLiveIn>(getStepValue())->getValue());
- auto *StartC =
- dyn_cast<ConstantInt>(cast<VPLiveIn>(getStartValue())->getValue());
+ ;
+ auto *StepC = dyn_cast<ConstantInt>(Step->getValue());
+ auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getValue());
return StartC && StartC->isZero() && StepC && StepC->isOne() &&
getScalarType() == getRegion()->getCanonicalIVType();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9ae510d499626..f5044c8621c66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -79,7 +79,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
for (VPValue *Op : PhiR->operands())
NewRecipe->addOperand(Op);
} else {
- VPValue *Start = Plan.getOrAddLiveIn(II->getStartValue());
+ VPLiveIn *Start = Plan.getOrAddLiveIn(II->getStartValue());
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
// It is always safe to copy over the NoWrap and FastMath flags. In
@@ -736,7 +736,7 @@ static VPScalarIVStepsRecipe *
createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode,
FPMathOperator *FPBinOp, Instruction *TruncI,
- VPValue *StartV, VPValue *Step, DebugLoc DL,
+ VPLiveIn *StartV, VPValue *Step, DebugLoc DL,
VPBuilder &Builder) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
@@ -791,7 +791,7 @@ static VPValue *
scalarizeVPWidenPointerInduction(VPWidenPointerInductionRecipe *PtrIV,
VPlan &Plan, VPBuilder &Builder) {
const InductionDescriptor &ID = PtrIV->getInductionDescriptor();
- VPValue *StartV = Plan.getConstantInt(ID.getStep()->getType(), 0);
+ VPLiveIn *StartV = Plan.getConstantInt(ID.getStep()->getType(), 0);
VPValue *StepV = PtrIV->getOperand(1);
VPScalarIVStepsRecipe *Steps = createScalarIVSteps(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
@@ -996,7 +996,7 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
const InductionDescriptor &ID = WideIV->getInductionDescriptor();
- VPValue *Start = WideIV->getStartValue();
+ VPLiveIn *Start = WideIV->getStartValue();
VPValue *Step = WideIV->getStepValue();
EndValue = B.createDerivedIV(
ID.getKind(), dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()),
@@ -4643,7 +4643,7 @@ void VPlanTransforms::materializeVectorTripCount(VPlan &Plan,
bool RequiresScalarEpilogue) {
VPValue &VectorTC = Plan.getVectorTripCount();
assert(isa<VPSymbolicValue>(VectorTC) &&
- "vector-trip-count must be a live-in");
+ "vector-trip-count must be symbolic");
// There's nothing to do if there are no users of the vector trip count or its
// IR value has already been set.
if (VectorTC.getNumUsers() == 0 || VectorTC.getUnderlyingValue())
@@ -5065,7 +5065,7 @@ static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
if (WideIntOrFp && WideIntOrFp->getTruncInst())
return nullptr;
- VPValue *Start = WideIV->getStartValue();
+ VPLiveIn *Start = WideIV->getStartValue();
VPValue *Step = WideIV->getStepValue();
const InductionDescriptor &ID = WideIV->getInductionDescriptor();
VPValue *EndValue = VectorTC;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 4fb15af08037d..bbadc292c2731 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -201,12 +201,11 @@ struct VPSymbolicValue : public VPValue {
}
};
-/// A VPValue defined by a recipe that produces multiple values.
+/// A VPValue defined by a recipe that produces one or more values.
class VPDefValue : public VPValue {
friend class VPValue;
friend class VPDef;
- /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the
- /// VPValue is not defined by any recipe modeled in VPlan.
+ /// Pointer to the VPDef that defines this VPValue.
VPDef *Def;
public:
>From 93d1d1afec812097950a0798833f9a2d13280a16 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 28 Dec 2025 15:51:26 +0000
Subject: [PATCH 3/4] !fixup re-added getLiveInIRValue
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 ++++
llvm/lib/Transforms/Vectorize/VPlan.h | 6 +++---
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 ++++++++--------
.../lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
llvm/lib/Transforms/Vectorize/VPlanValue.h | 3 +++
6 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 885dcb92d8a74..a1e0458ee5f0d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -138,6 +138,10 @@ const VPRecipeBase *VPValue::getDefiningRecipe() const {
return cast<VPRecipeBase>(DefValue->Def);
}
+Value *VPValue::getLiveInIRValue() const {
+ return cast<VPLiveIn>(this)->getValue();
+}
+
Type *VPLiveIn::getType() const { return getUnderlyingValue()->getType(); }
VPDefValue::VPDefValue(VPDef *Def, Value *UV)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d26ebd6f32718..08e4a2d48f758 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1718,7 +1718,7 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(isa<Function>(
- cast<VPLiveIn>(getOperand(getNumOperands() - 1))->getValue()) &&
+ getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
}
@@ -1740,7 +1740,7 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
Function *getCalledScalarFunction() const {
return cast<Function>(
- cast<VPLiveIn>(getOperand(getNumOperands() - 1))->getValue());
+ getOperand(getNumOperands() - 1)->getLiveInIRValue());
}
operand_range args() { return drop_end(operands()); }
@@ -3139,7 +3139,7 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
"Expected an add reduction");
assert(getNumOperands() >= 3 && "Expected at least three operands");
[[maybe_unused]] auto *SubConst =
- dyn_cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue());
+ dyn_cast<ConstantInt>(getOperand(2)->getLiveInIRValue());
assert(SubConst && SubConst->getValue() == 0 &&
Sub->getOpcode() == Instruction::Sub && "Expected a negating sub");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 73678d6f65b48..7247e0b8568a8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -170,7 +170,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenRecipe *R) {
case Instruction::ExtractValue: {
assert(R->getNumOperands() == 2 && "expected single level extractvalue");
auto *StructTy = cast<StructType>(inferScalarType(R->getOperand(0)));
- auto *CI = cast<ConstantInt>(cast<VPLiveIn>(R->getOperand(1))->getValue());
+ auto *CI = cast<ConstantInt>(R->getOperand(1)->getLiveInIRValue());
return StructTy->getTypeAtIndex(CI->getZExtValue());
}
default:
@@ -223,7 +223,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPReplicateRecipe *R) {
switch (Opcode) {
case Instruction::Call: {
unsigned CallIdx = R->getNumOperands() - (R->isPredicated() ? 2 : 1);
- return cast<Function>(cast<VPLiveIn>(R->getOperand(CallIdx))->getValue())
+ return cast<Function>(R->getOperand(CallIdx)->getLiveInIRValue())
->getReturnType();
}
case Instruction::Select: {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 19c89168b37a1..550a784325202 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -395,7 +395,7 @@ VPUnrollPartAccessor<PartOpIdx>::getUnrollPartOperand(const VPUser &U) const {
template <unsigned PartOpIdx>
unsigned VPUnrollPartAccessor<PartOpIdx>::getUnrollPart(const VPUser &U) const {
if (auto *UnrollPartOp = getUnrollPartOperand(U))
- return cast<ConstantInt>(cast<VPLiveIn>(UnrollPartOp)->getValue())
+ return cast<ConstantInt>(UnrollPartOp->getLiveInIRValue())
->getZExtValue();
return 0;
}
@@ -580,7 +580,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Name);
ElementCount EC = State.VF.multiplyCoefficientBy(
- cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue())
+ cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
->getZExtValue());
auto *PredTy = VectorType::get(Builder.getInt1Ty(), EC);
return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
@@ -698,7 +698,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
// If this start vector is scaled then it should produce a vector with fewer
// elements than the VF.
ElementCount VF = State.VF.divideCoefficientBy(
- cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue())
+ cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
->getZExtValue());
auto *Iden = Builder.CreateVectorSplat(VF, State.get(getOperand(1), true));
return Builder.CreateInsertElement(Iden, State.get(getOperand(0), true),
@@ -743,7 +743,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
State.get(getOperand(3 + Part)));
Value *Start = State.get(getOperand(1), true);
- Value *Sentinel = cast<VPLiveIn>(getOperand(2))->getValue();
+ Value *Sentinel = getOperand(2)->getLiveInIRValue();
return createFindLastIVReduction(Builder, ReducedPartRdx, RK, Start,
Sentinel);
}
@@ -1080,7 +1080,7 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask: {
Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));
unsigned Multiplier =
- cast<ConstantInt>(cast<VPLiveIn>(getOperand(2))->getValue())
+ cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
@@ -2130,7 +2130,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
case Instruction::ExtractValue: {
assert(getNumOperands() == 2 && "expected single level extractvalue");
Value *Op = State.get(getOperand(0));
- auto *CI = cast<ConstantInt>(cast<VPLiveIn>(getOperand(1))->getValue());
+ auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
Value *Extract = Builder.CreateExtractValue(Op, CI->getZExtValue());
State.set(this, Extract);
break;
@@ -3214,7 +3214,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
return 0;
case Instruction::Call: {
auto *CalledFn = cast<Function>(
- cast<VPLiveIn>(getOperand(getNumOperands() - 1))->getValue());
+ getOperand(getNumOperands() - 1)->getLiveInIRValue());
SmallVector<const VPValue *> ArgOps(drop_end(operands()));
SmallVector<Type *, 4> Tys;
@@ -4288,7 +4288,7 @@ void VPWidenCanonicalIVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
// Create a vector from the initial value.
- auto *VectorInit = cast<VPLiveIn>(getStartValue())->getValue();
+ auto *VectorInit = getStartValue()->getLiveInIRValue();
Type *VecTy = State.VF.isScalar()
? VectorInit->getType()
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f5044c8621c66..2c9c058da9468 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4207,7 +4207,7 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
auto *VectorPreheader = Plan.getVectorPreheader();
for (VPValue *VPV : VPValues) {
if (vputils::onlyScalarValuesUsed(VPV) ||
- (isa<VPLiveIn>(VPV) && isa<Constant>(cast<VPLiveIn>(VPV)->getValue())))
+ (isa<VPLiveIn>(VPV) && isa<Constant>(VPV->getLiveInIRValue())))
continue;
// Add explicit broadcast at the insert point that dominates all users.
@@ -4521,7 +4521,7 @@ void VPlanTransforms::materializeConstantVectorTripCount(
// TODO: Compute vector trip counts for loops requiring a scalar epilogue and
// tail-folded loops.
ScalarEvolution &SE = *PSE.getSE();
- auto *TCScev = SE.getSCEV(cast<VPLiveIn>(TC)->getValue());
+ auto *TCScev = SE.getSCEV(TC->getLiveInIRValue());
if (!isa<SCEVConstant>(TCScev))
return;
const SCEV *VFxUF = SE.getElementCount(TCScev->getType(), BestVF * BestUF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index bbadc292c2731..10c7655bb59f3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -70,6 +70,9 @@ class LLVM_ABI_FOR_TEST VPValue {
/// Return the underlying Value attached to this VPValue.
Value *getUnderlyingValue() const { return UnderlyingVal; }
+ /// Return the underlying IR value for a VPLiveIn.
+ Value *getLiveInIRValue() const;
+
/// An enumeration for keeping track of the concrete subclass of VPValue that
/// are actually instantiated.
enum {
>From 4af51c6418d0de8f523ae0ed51aa1727aef57a6b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 28 Dec 2025 16:06:51 +0000
Subject: [PATCH 4/4] !fixup adjust naming
---
.../Vectorize/LoopVectorizationPlanner.h | 2 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 34 ++++----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 32 ++++----
llvm/lib/Transforms/Vectorize/VPlan.h | 82 +++++++++----------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 6 +-
.../Vectorize/VPlanConstruction.cpp | 4 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 6 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 30 +++----
.../Transforms/Vectorize/VPlanTransforms.cpp | 30 +++----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 32 ++++----
.../Vectorize/VPlanPatternMatchTest.cpp | 2 +-
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
.../Vectorize/VPlanVerifierTest.cpp | 16 ++--
15 files changed, 140 insertions(+), 148 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index eb6ca43aec490..ddef767f6937c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -296,7 +296,7 @@ class VPBuilder {
/// induction with \p Start and \p Step values, using \p Start + \p Current *
/// \p Step.
VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
- FPMathOperator *FPBinOp, VPLiveIn *Start,
+ FPMathOperator *FPBinOp, VPIRValue *Start,
VPValue *Current, VPValue *Step,
const Twine &Name = "") {
return tryInsertInstruction(
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e493b7861e2d6..4db6f28a1de35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4073,11 +4073,9 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
} else {
auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
Function *CalledFn =
- WidenCall
- ? WidenCall->getCalledScalarFunction()
- : cast<Function>(
- cast<VPLiveIn>(R->getOperand(R->getNumOperands() - 1))
- ->getValue());
+ WidenCall ? WidenCall->getCalledScalarFunction()
+ : cast<Function>(R->getOperand(R->getNumOperands() - 1)
+ ->getLiveInIRValue());
Name = CalledFn->getName();
}
OS << " call to " << Name;
@@ -4278,8 +4276,7 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
}
case VPInstruction::ActiveLaneMask: {
unsigned Multiplier =
- cast<ConstantInt>(
- cast<VPLiveIn>(VPI->getOperand(2))->getValue())
+ cast<ConstantInt>(VPI->getOperand(2)->getLiveInIRValue())
->getZExtValue();
C += VPI->cost(VF * Multiplier, CostCtx);
break;
@@ -7287,7 +7284,7 @@ static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
"RdxResult must be ComputeFindIVResult");
VPValue *StartVPV = RdxResult->getOperand(1);
match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
- return cast<VPLiveIn>(StartVPV)->getValue();
+ return StartVPV->getLiveInIRValue();
}
// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
@@ -7321,7 +7318,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
[[maybe_unused]] Value *StartV =
- cast<VPLiveIn>(EpiRedResult->getOperand(1))->getValue();
+ EpiRedResult->getOperand(1)->getLiveInIRValue();
auto *Cmp = cast<ICmpInst>(MainResumeValue);
assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
"AnyOf expected to start with ICMP_NE");
@@ -7331,7 +7328,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = Cmp->getOperand(0);
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
Value *StartV = getStartValueFromReductionResult(EpiRedResult);
- Value *SentinelV = cast<VPLiveIn>(EpiRedResult->getOperand(2))->getValue();
+ Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
using namespace llvm::PatternMatch;
Value *Cmp, *OrigResumeV, *CmpOp;
[[maybe_unused]] bool IsExpectedPattern =
@@ -7426,7 +7423,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
DenseMap<const SCEV *, Value *> ExpandedSCEVs =
VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
if (!ILV.getTripCount()) {
- ILV.setTripCount(cast<VPLiveIn>(BestVPlan.getTripCount())->getValue());
+ ILV.setTripCount(BestVPlan.getTripCount()->getLiveInIRValue());
} else {
assert(VectorizingEpilogue && "should only re-use the existing trip "
"count during epilogue vectorization");
@@ -7749,7 +7746,7 @@ VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
auto *WidenIV = cast<VPWidenIntOrFpInductionRecipe>(
VPI->getOperand(0)->getDefiningRecipe());
PHINode *Phi = WidenIV->getPHINode();
- VPLiveIn *Start = WidenIV->getStartValue();
+ VPIRValue *Start = WidenIV->getStartValue();
const InductionDescriptor &IndDesc = WidenIV->getInductionDescriptor();
// It is always safe to copy over the NoWrap and FastMath flags. In
@@ -8919,8 +8916,8 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) {
Value *Step = State.get(getStepValue(), VPLane(0));
Value *Index = State.get(getOperand(1), VPLane(0));
Value *DerivedIV = emitTransformedIndex(
- State.Builder, Index, cast<VPLiveIn>(getStartValue())->getValue(), Step,
- Kind, cast_if_present<BinaryOperator>(FPBinOp));
+ State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
+ cast_if_present<BinaryOperator>(FPBinOp));
DerivedIV->setName(Name);
State.set(this, DerivedIV, VPLane(0));
}
@@ -9268,8 +9265,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
continue;
VPValue *OrigStart = VPI->getOperand(1);
- if (isGuaranteedNotToBeUndefOrPoison(
- cast<VPLiveIn>(OrigStart)->getValue()))
+ if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
continue;
VPInstruction *Freeze =
Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
@@ -9392,7 +9388,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
->getIncomingValueForBlock(L->getLoopPreheader());
RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
- Value *StartV = cast<VPLiveIn>(RdxResult->getOperand(1))->getValue();
+ Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
@@ -9417,7 +9413,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
if (auto *I = dyn_cast<Instruction>(Cmp))
InstsToMove.push_back(I);
- Value *Sentinel = cast<VPLiveIn>(RdxResult->getOperand(2))->getValue();
+ Value *Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
if (auto *I = dyn_cast<Instruction>(ResumeV))
InstsToMove.push_back(I);
@@ -9455,7 +9451,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
auto *VPI = dyn_cast<VPInstruction>(&R);
if (VPI && VPI->getOpcode() == Instruction::Freeze) {
VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
- ToFrozen.lookup(cast<VPLiveIn>(VPI->getOperand(0))->getValue())));
+ ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a1e0458ee5f0d..26cb6b1463e20 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -124,14 +124,14 @@ void VPDef::dump() const {
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
- auto *DefValue = dyn_cast<VPDefValue>(this);
+ auto *DefValue = dyn_cast<VPRecipeValue>(this);
if (!DefValue)
return nullptr;
return cast<VPRecipeBase>(DefValue->Def);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *DefValue = dyn_cast<VPDefValue>(this);
+ auto *DefValue = dyn_cast<VPRecipeValue>(this);
if (!DefValue)
return nullptr;
assert(DefValue);
@@ -139,18 +139,18 @@ const VPRecipeBase *VPValue::getDefiningRecipe() const {
}
Value *VPValue::getLiveInIRValue() const {
- return cast<VPLiveIn>(this)->getValue();
+ return cast<VPIRValue>(this)->getValue();
}
-Type *VPLiveIn::getType() const { return getUnderlyingValue()->getType(); }
+Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPDefValue::VPDefValue(VPDef *Def, Value *UV)
+VPRecipeValue::VPRecipeValue(VPDef *Def, Value *UV)
: VPValue(VPVDefValueSC, UV, nullptr), Def(Def) {
- assert(Def && "VPDefValue requires a defining recipe");
+ assert(Def && "VPRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPDefValue::~VPDefValue() {
+VPRecipeValue::~VPRecipeValue() {
assert(Users.empty() && "trying to delete a VPValue with remaining users");
if (Def)
Def->removeDefinedValue(this);
@@ -249,7 +249,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
CurrentParentLoop(CurrentParentLoop), TypeAnalysis(*Plan), VPDT(*Plan) {}
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
- if (isa<VPLiveIn, VPSymbolicValue>(Def))
+ if (isa<VPIRValue, VPSymbolicValue>(Def))
return Def->getUnderlyingValue();
if (hasScalarValue(Def, Lane))
@@ -282,7 +282,7 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
if (NeedsScalar) {
- assert((VF.isScalar() || isa<VPLiveIn, VPSymbolicValue>(Def) ||
+ assert((VF.isScalar() || isa<VPIRValue, VPSymbolicValue>(Def) ||
hasVectorValue(Def) || !vputils::onlyFirstLaneUsed(Def) ||
(hasScalarValue(Def, VPLane(0)) &&
Data.VPV2Scalars[Def].size() == 1)) &&
@@ -304,7 +304,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
};
if (!hasScalarValue(Def, {0})) {
- assert((isa<VPLiveIn, VPSymbolicValue>(Def)) && "expected a live-in");
+ assert((isa<VPIRValue, VPSymbolicValue>(Def)) && "expected a live-in");
Value *IRV = Def->getUnderlyingValue();
Value *B = GetBroadcastInstrs(IRV);
set(Def, B);
@@ -1072,7 +1072,7 @@ void VPlan::printLiveIns(raw_ostream &O) const {
O << "\n";
if (TripCount) {
- if (isa<VPLiveIn>(TripCount))
+ if (isa<VPIRValue>(TripCount))
O << "Live-in ";
TripCount->printAsOperand(O, SlotTracker);
O << " = original trip-count";
@@ -1188,7 +1188,7 @@ VPlan *VPlan::duplicate() {
// Create VPlan, clone live-ins and remap operands in the cloned blocks.
auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
- for (VPLiveIn *OldLiveIn : getLiveIns()) {
+ for (VPIRValue *OldLiveIn : getLiveIns()) {
Old2NewVPValues[OldLiveIn] = NewPlan->getOrAddLiveIn(OldLiveIn->getValue());
}
Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount;
@@ -1198,7 +1198,7 @@ VPlan *VPlan::duplicate() {
NewPlan->BackedgeTakenCount = new VPSymbolicValue();
Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount;
}
- if (auto *LI = dyn_cast_or_null<VPLiveIn>(TripCount))
+ if (auto *LI = dyn_cast_or_null<VPIRValue>(TripCount))
Old2NewVPValues[LI] = NewPlan->getOrAddLiveIn(LI->getValue());
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
@@ -1467,7 +1467,7 @@ void VPSlotTracker::assignName(const VPValue *V) {
const auto &[A, _] = VPValue2Name.try_emplace(V, BaseName);
// Integer or FP constants with different types will result in he same string
// due to stripping types.
- if (isa<VPLiveIn>(V) && isa<ConstantInt, ConstantFP>(UV))
+ if (isa<VPIRValue>(V) && isa<ConstantInt, ConstantFP>(UV))
return;
// If it is already used by C > 0 other VPValues, increase the version counter
@@ -1744,7 +1744,7 @@ bool llvm::canConstantBeExtended(const APInt *C, Type *NarrowType,
TargetTransformInfo::OperandValueInfo
VPCostContext::getOperandInfo(VPValue *V) const {
- if (auto *LI = dyn_cast<VPLiveIn>(V))
+ if (auto *LI = dyn_cast<VPIRValue>(V))
return TTI::getOperandInfo(LI->getValue());
return {};
@@ -1775,7 +1775,7 @@ InstructionCost VPCostContext::getScalarizationOverhead(
SmallPtrSet<const VPValue *, 4> UniqueOperands;
SmallVector<Type *> Tys;
for (auto *Op : Operands) {
- if (isa<VPLiveIn>(Op) ||
+ if (isa<VPIRValue>(Op) ||
(!AlwaysIncludeReplicatingR &&
isa<VPReplicateRecipe, VPPredInstPHIRecipe>(Op)) ||
(isa<VPReplicateRecipe>(Op) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 08e4a2d48f758..a999d99671591 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -528,15 +528,15 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPDefValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPDefValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPDefValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPDefID()) {
@@ -1717,9 +1717,9 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
- assert(isa<Function>(
- getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
- "last operand must be the called function");
+ assert(
+ isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
+ "last operand must be the called function");
}
~VPWidenCallRecipe() override = default;
@@ -1739,8 +1739,7 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
VPCostContext &Ctx) const override;
Function *getCalledScalarFunction() const {
- return cast<Function>(
- getOperand(getNumOperands() - 1)->getLiveInIRValue());
+ return cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
}
operand_range args() { return drop_end(operands()); }
@@ -2148,7 +2147,7 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
void execute(VPTransformState &State) override = 0;
- VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+ VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
/// Returns the step value of the induction.
VPValue *getStepValue() { return getOperand(1); }
@@ -2206,7 +2205,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
bool isUnrolled() const { return getNumOperands() == 5; }
public:
- VPWidenIntOrFpInductionRecipe(PHINode *IV, VPLiveIn *Start, VPValue *Step,
+ VPWidenIntOrFpInductionRecipe(PHINode *IV, VPIRValue *Start, VPValue *Step,
VPValue *VF, const InductionDescriptor &IndDesc,
const VPIRFlags &Flags, DebugLoc DL)
: VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start,
@@ -2215,7 +2214,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
addOperand(VF);
}
- VPWidenIntOrFpInductionRecipe(PHINode *IV, VPLiveIn *Start, VPValue *Step,
+ VPWidenIntOrFpInductionRecipe(PHINode *IV, VPIRValue *Start, VPValue *Step,
VPValue *VF, const InductionDescriptor &IndDesc,
TruncInst *Trunc, const VPIRFlags &Flags,
DebugLoc DL)
@@ -2245,7 +2244,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
"expandVPWidenIntOrFpInductionRecipe");
}
- VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+ VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
VPValue *getSplatVFValue() {
// If the recipe has been unrolled return the VPValue for the induction
@@ -2634,7 +2633,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (Instruction *Inst = IG->getMember(I)) {
if (Inst->getType()->isVoidTy())
continue;
- new VPDefValue(this, Inst);
+ new VPRecipeValue(this, Inst);
}
for (auto *SV : StoredValues)
@@ -3357,13 +3356,13 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPDefValue {
+ public VPRecipeValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, bool Reverse,
const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
Reverse, Metadata, DL),
- VPDefValue(this, &Load) {
+ VPRecipeValue(this, &Load) {
setMask(Mask);
}
@@ -3399,13 +3398,13 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
/// using the address to load from, the explicit vector length and an optional
/// mask.
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
- public VPDefValue {
+ public VPRecipeValue {
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
VPValue *Mask)
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
{Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
L.getDebugLoc()),
- VPDefValue(this, &getIngredient()) {
+ VPRecipeValue(this, &getIngredient()) {
setMask(Mask);
}
@@ -3570,7 +3569,7 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
/// canonical induction variable.
class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
public:
- VPCanonicalIVPHIRecipe(VPLiveIn *StartV, DebugLoc DL)
+ VPCanonicalIVPHIRecipe(VPIRValue *StartV, DebugLoc DL)
: VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV, DL) {}
~VPCanonicalIVPHIRecipe() override = default;
@@ -3588,7 +3587,7 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
"scalar phi recipe");
}
- VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+ VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
/// Returns the scalar type of the induction.
Type *getScalarType() const { return getStartValue()->getType(); }
@@ -3746,7 +3745,7 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
std::string Name;
public:
- VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPLiveIn *Start,
+ VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPIRValue *Start,
VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
const Twine &Name = "")
: VPDerivedIVRecipe(
@@ -3755,8 +3754,8 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
Start, CanonicalIV, Step, Name) {}
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
- const FPMathOperator *FPBinOp, VPLiveIn *Start, VPValue *IV,
- VPValue *Step, const Twine &Name = "")
+ const FPMathOperator *FPBinOp, VPIRValue *Start,
+ VPValue *IV, VPValue *Step, const Twine &Name = "")
: VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, IV, Step}), Kind(Kind),
FPBinOp(FPBinOp), Name(Name.str()) {}
@@ -3782,7 +3781,7 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
Type *getScalarType() const { return getStartValue()->getType(); }
- VPLiveIn *getStartValue() const { return cast<VPLiveIn>(getOperand(0)); }
+ VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
VPValue *getStepValue() const { return getOperand(2); }
/// Returns true if the recipe only uses the first lane of operand \p Op.
@@ -4348,8 +4347,8 @@ class VPlan {
VPSymbolicValue VFxUF;
/// Contains all the external definitions created for this VPlan, as a mapping
- /// from IR Values to VPLiveIns.
- SmallMapVector<Value *, VPLiveIn *, 16> LiveIns;
+ /// from IR Values to VPIRValues.
+ SmallMapVector<Value *, VPIRValue *, 16> LiveIns;
/// Blocks allocated and owned by the VPlan. They will be deleted once the
/// VPlan is destroyed.
@@ -4544,42 +4543,43 @@ class VPlan {
/// Gets the live-in VPValue for \p V or adds a new live-in (if none exists
/// yet) for \p V.
- VPLiveIn *getOrAddLiveIn(Value *V) {
+ VPIRValue *getOrAddLiveIn(Value *V) {
assert(V && "Trying to get or add the VPValue of a null Value");
auto [It, Inserted] = LiveIns.try_emplace(V);
if (Inserted)
- It->second = new VPLiveIn(V);
+ It->second = new VPIRValue(V);
- assert(isa<VPLiveIn>(It->second) && "Only VPLiveIns should be in mapping");
+ assert(isa<VPIRValue>(It->second) &&
+ "Only VPIRValues should be in mapping");
return It->second;
}
- /// Return a VPLiveIn wrapping i1 true.
- VPLiveIn *getTrue() { return getConstantInt(1, 1); }
+ /// Return a VPIRValue wrapping i1 true.
+ VPIRValue *getTrue() { return getConstantInt(1, 1); }
- /// Return a VPLiveIn wrapping i1 false.
- VPLiveIn *getFalse() { return getConstantInt(1, 0); }
+ /// Return a VPIRValue wrapping i1 false.
+ VPIRValue *getFalse() { return getConstantInt(1, 0); }
- /// Return a VPLiveIn wrapping a ConstantInt with the given type and value.
- VPLiveIn *getConstantInt(Type *Ty, uint64_t Val, bool IsSigned = false) {
+ /// Return a VPIRValue wrapping a ConstantInt with the given type and value.
+ VPIRValue *getConstantInt(Type *Ty, uint64_t Val, bool IsSigned = false) {
return getOrAddLiveIn(ConstantInt::get(Ty, Val, IsSigned));
}
- /// Return a VPLiveIn wrapping a ConstantInt with the given bitwidth and
+ /// Return a VPIRValue wrapping a ConstantInt with the given bitwidth and
/// value.
- VPLiveIn *getConstantInt(unsigned BitWidth, uint64_t Val,
- bool IsSigned = false) {
+ VPIRValue *getConstantInt(unsigned BitWidth, uint64_t Val,
+ bool IsSigned = false) {
return getConstantInt(APInt(BitWidth, Val, IsSigned));
}
- /// Return a VPLiveIn wrapping a ConstantInt with the given APInt value.
- VPLiveIn *getConstantInt(const APInt &Val) {
+ /// Return a VPIRValue wrapping a ConstantInt with the given APInt value.
+ VPIRValue *getConstantInt(const APInt &Val) {
return getOrAddLiveIn(ConstantInt::get(getContext(), Val));
}
- /// Return the live-in VPLiveIn for \p V, if there is one or nullptr
+ /// Return the live-in VPIRValue for \p V, if there is one or nullptr
/// otherwise.
- VPLiveIn *getLiveIn(Value *V) const { return LiveIns.lookup(V); }
+ VPIRValue *getLiveIn(Value *V) const { return LiveIns.lookup(V); }
/// Return the list of live-in VPValues available in the VPlan.
auto getLiveIns() const { return LiveIns.values(); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 7247e0b8568a8..a5ed7137ae534 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -36,7 +36,7 @@ VPTypeAnalysis::VPTypeAnalysis(const VPlan &Plan) : Ctx(Plan.getContext()) {
// If there's no canonical IV, retrieve the type from the trip count
// expression.
auto *TC = Plan.getTripCount();
- if (auto *TCLI = dyn_cast<VPLiveIn>(TC)) {
+ if (auto *TCLI = dyn_cast<VPIRValue>(TC)) {
CanonicalIVTy = TCLI->getType();
return;
}
@@ -265,7 +265,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
if (Type *CachedTy = CachedTypes.lookup(V))
return CachedTy;
- if (auto *LI = dyn_cast<VPLiveIn>(V))
+ if (auto *LI = dyn_cast<VPIRValue>(V))
return LI->getType();
if (isa<VPSymbolicValue>(V)) {
@@ -449,7 +449,7 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
// FIXME: Might need some motivation why these values are ignored. If
// for example an argument is used inside the loop it will increase the
// register pressure (so shouldn't we add it to LoopInvariants).
- auto *LI = dyn_cast<VPLiveIn>(U);
+ auto *LI = dyn_cast<VPIRValue>(U);
if (!DefR && (!LI || !isa<Instruction>(LI->getValue())))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4ae1fc239d9be..c3987dc14d1a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -599,7 +599,7 @@ VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
/// Creates a VPWidenIntOrFpInductionRecipe or VPWidenPointerInductionRecipe
/// for \p Phi based on \p IndDesc.
static VPHeaderPHIRecipe *
-createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPLiveIn *Start,
+createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
const InductionDescriptor &IndDesc, VPlan &Plan,
PredicatedScalarEvolution &PSE, Loop &OrigLoop,
DebugLoc DL) {
@@ -660,7 +660,7 @@ void VPlanTransforms::createHeaderPhiRecipes(
"Must have 2 operands for header phis");
// Extract common values once.
- VPLiveIn *Start = cast<VPLiveIn>(PhiR->getOperand(0));
+ VPIRValue *Start = cast<VPIRValue>(PhiR->getOperand(0));
VPValue *BackedgeValue = PhiR->getOperand(1);
if (FixedOrderRecurrences.contains(Phi)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 5e30683b66c3d..33bf8244aa130 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -106,7 +106,7 @@ template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
int_pred_ty() : P() {}
bool match(VPValue *VPV) const {
- auto *LI = dyn_cast<VPLiveIn>(VPV);
+ auto *LI = dyn_cast<VPIRValue>(VPV);
if (!LI)
return false;
assert(!LI->getType()->isVectorTy() && "Unexpected vector live-in");
@@ -180,7 +180,7 @@ struct bind_apint {
bind_apint(const APInt *&Res) : Res(Res) {}
bool match(VPValue *VPV) const {
- auto *LI = dyn_cast<VPLiveIn>(VPV);
+ auto *LI = dyn_cast<VPIRValue>(VPV);
if (!LI)
return false;
assert(!LI->getType()->isVectorTy() && "Unexpected vector live-in");
@@ -903,7 +903,7 @@ m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
struct live_in_vpvalue {
template <typename ITy> bool match(ITy *V) const {
- return isa<VPLiveIn, VPSymbolicValue>(V);
+ return isa<VPIRValue, VPSymbolicValue>(V);
}
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 550a784325202..c5e1714866053 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -395,8 +395,7 @@ VPUnrollPartAccessor<PartOpIdx>::getUnrollPartOperand(const VPUser &U) const {
template <unsigned PartOpIdx>
unsigned VPUnrollPartAccessor<PartOpIdx>::getUnrollPart(const VPUser &U) const {
if (auto *UnrollPartOp = getUnrollPartOperand(U))
- return cast<ConstantInt>(UnrollPartOp->getLiveInIRValue())
- ->getZExtValue();
+ return cast<ConstantInt>(UnrollPartOp->getLiveInIRValue())->getZExtValue();
return 0;
}
@@ -535,7 +534,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
case Instruction::ExtractElement: {
assert(State.VF.isVector() && "Only extract elements from vectors");
- if (auto *IdxLI = dyn_cast<VPLiveIn>(getOperand(1))) {
+ if (auto *IdxLI = dyn_cast<VPIRValue>(getOperand(1))) {
unsigned IdxToExtract =
cast<ConstantInt>(IdxLI->getValue())->getZExtValue();
return State.get(getOperand(0), VPLane(IdxToExtract));
@@ -580,8 +579,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Name);
ElementCount EC = State.VF.multiplyCoefficientBy(
- cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
- ->getZExtValue());
+ cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue());
auto *PredTy = VectorType::get(Builder.getInt1Ty(), EC);
return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
{PredTy, ScalarTC->getType()},
@@ -698,8 +696,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
// If this start vector is scaled then it should produce a vector with fewer
// elements than the VF.
ElementCount VF = State.VF.divideCoefficientBy(
- cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
- ->getZExtValue());
+ cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue());
auto *Iden = Builder.CreateVectorSplat(VF, State.get(getOperand(1), true));
return Builder.CreateInsertElement(Iden, State.get(getOperand(0), true),
Builder.getInt32(0));
@@ -1080,8 +1077,7 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ActiveLaneMask: {
Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));
unsigned Multiplier =
- cast<ConstantInt>(getOperand(2)->getLiveInIRValue())
- ->getZExtValue();
+ cast<ConstantInt>(getOperand(2)->getLiveInIRValue())->getZExtValue();
Type *RetTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF * Multiplier);
IntrinsicCostAttributes Attrs(Intrinsic::get_active_lane_mask, RetTy,
{ArgTy, ArgTy});
@@ -1507,7 +1503,7 @@ void VPIRInstruction::extractLastLaneOfLastPartOfFirstOperand(
"can only update exiting operands to phi nodes");
assert(getNumOperands() > 0 && "must have at least one operand");
VPValue *Exiting = getOperand(0);
- if (isa<VPLiveIn>(Exiting))
+ if (isa<VPIRValue>(Exiting))
return;
Exiting = Builder.createNaryOp(VPInstruction::ExtractLastPart, Exiting);
@@ -1872,7 +1868,7 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
// a multiply, and add that into the cost.
InstructionCost MulCost =
Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
- if (auto *IncAmtLI = dyn_cast<VPLiveIn>(IncAmt)) {
+ if (auto *IncAmtLI = dyn_cast<VPIRValue>(IncAmt)) {
ConstantInt *CI = dyn_cast<ConstantInt>(IncAmtLI->getValue());
if (CI && CI->getZExtValue() == 1)
@@ -1973,7 +1969,7 @@ InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
llvm::CmpPredicate Pred;
if (!match(getOperand(0), m_Cmp(Pred, m_VPValue(), m_VPValue())))
- if (auto *LiveIn = dyn_cast<VPLiveIn>(getOperand(0)))
+ if (auto *LiveIn = dyn_cast<VPIRValue>(getOperand(0)))
if (auto *Cmp = dyn_cast<CmpInst>(LiveIn->getValue()))
Pred = Cmp->getPredicate();
return Ctx.TTI.getCmpSelInstrCost(
@@ -2286,7 +2282,7 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,
// For Z/Sext, get the context from the operand.
else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
Opcode == Instruction::FPExt) {
- if (isa<VPLiveIn>(Operand))
+ if (isa<VPIRValue>(Operand))
CCH = TTI::CastContextHint::Normal;
else if (auto *Recipe = Operand->getDefiningRecipe()) {
VPValue *ReverseOp;
@@ -2348,7 +2344,7 @@ bool VPWidenIntOrFpInductionRecipe::isCanonical() const {
// The step may be defined by a recipe in the preheader (e.g. if it requires
// SCEV expansion), but for the canonical induction the step is required to be
// 1, which is represented as live-in.
- const VPLiveIn *Step = dyn_cast<VPLiveIn>(getStepValue());
+ const VPIRValue *Step = dyn_cast<VPIRValue>(getStepValue());
if (!Step)
return false;
;
@@ -3213,8 +3209,8 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
// instruction cost.
return 0;
case Instruction::Call: {
- auto *CalledFn = cast<Function>(
- getOperand(getNumOperands() - 1)->getLiveInIRValue());
+ auto *CalledFn =
+ cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
SmallVector<const VPValue *> ArgOps(drop_end(operands()));
SmallVector<Type *, 4> Tys;
@@ -3891,7 +3887,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
// TODO: Also manage existing metadata using VPIRMetadata.
Group->addMetadata(NewLoad);
- ArrayRef<VPDefValue *> VPDefs = definedValues();
+ ArrayRef<VPRecipeValue *> VPDefs = definedValues();
if (VecTy->isScalableTy()) {
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2c9c058da9468..e5a04fbaf61b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -79,7 +79,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
for (VPValue *Op : PhiR->operands())
NewRecipe->addOperand(Op);
} else {
- VPLiveIn *Start = Plan.getOrAddLiveIn(II->getStartValue());
+ VPIRValue *Start = Plan.getOrAddLiveIn(II->getStartValue());
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
// It is always safe to copy over the NoWrap and FastMath flags. In
@@ -736,7 +736,7 @@ static VPScalarIVStepsRecipe *
createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
Instruction::BinaryOps InductionOpcode,
FPMathOperator *FPBinOp, Instruction *TruncI,
- VPLiveIn *StartV, VPValue *Step, DebugLoc DL,
+ VPIRValue *StartV, VPValue *Step, DebugLoc DL,
VPBuilder &Builder) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
@@ -791,7 +791,7 @@ static VPValue *
scalarizeVPWidenPointerInduction(VPWidenPointerInductionRecipe *PtrIV,
VPlan &Plan, VPBuilder &Builder) {
const InductionDescriptor &ID = PtrIV->getInductionDescriptor();
- VPLiveIn *StartV = Plan.getConstantInt(ID.getStep()->getType(), 0);
+ VPIRValue *StartV = Plan.getConstantInt(ID.getStep()->getType(), 0);
VPValue *StepV = PtrIV->getOperand(1);
VPScalarIVStepsRecipe *Steps = createScalarIVSteps(
Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr,
@@ -996,7 +996,7 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
const InductionDescriptor &ID = WideIV->getInductionDescriptor();
- VPLiveIn *Start = WideIV->getStartValue();
+ VPIRValue *Start = WideIV->getStartValue();
VPValue *Step = WideIV->getStepValue();
EndValue = B.createDerivedIV(
ID.getKind(), dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()),
@@ -1153,7 +1153,7 @@ static VPValue *tryToFoldLiveIns(VPSingleDefRecipe &R,
SmallVector<Value *, 4> Ops;
for (VPValue *Op : Operands) {
- if (!isa<VPLiveIn, VPSymbolicValue>(Op))
+ if (!isa<VPIRValue, VPSymbolicValue>(Op))
return nullptr;
Value *V = Op->getUnderlyingValue();
if (!V)
@@ -1229,7 +1229,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
// Fold PredPHI LiveIn -> LiveIn.
if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Def)) {
VPValue *Op = PredPHI->getOperand(0);
- if (isa<VPLiveIn>(Op))
+ if (isa<VPIRValue>(Op))
PredPHI->replaceAllUsesWith(Op);
}
@@ -1509,7 +1509,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
return;
// Hoist an invariant increment Y of a phi X, by having X start at Y.
- if (match(Def, m_c_Add(m_VPValue(X), m_VPValue(Y))) && isa<VPLiveIn>(Y) &&
+ if (match(Def, m_c_Add(m_VPValue(X), m_VPValue(Y))) && isa<VPIRValue>(Y) &&
isa<VPPhi>(X)) {
auto *Phi = cast<VPPhi>(X);
if (Phi->getOperand(1) != Def && match(Phi->getOperand(0), m_ZeroInt()) &&
@@ -1669,7 +1669,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
return false;
// Non-constant live-ins require broadcasts, while constants do not
// need explicit broadcasts.
- auto *LI = dyn_cast<VPLiveIn>(Op);
+ auto *LI = dyn_cast<VPIRValue>(Op);
bool LiveInNeedsBroadcast = LI && !isa<Constant>(LI->getValue());
auto *OpR = dyn_cast<VPReplicateRecipe>(Op);
return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
@@ -2584,7 +2584,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
}
VPBuilder Builder;
- if (isa<VPLiveIn>(Op))
+ if (isa<VPIRValue>(Op))
Builder.setInsertPoint(PH);
else
Builder.setInsertPoint(&R);
@@ -3897,7 +3897,7 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
}
VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx);
- if (!isa<VPLiveIn>(IncomingFromEarlyExit)) {
+ if (!isa<VPIRValue>(IncomingFromEarlyExit)) {
// Update the incoming value from the early exit.
VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {CondToEarlyExit},
@@ -4067,7 +4067,7 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
auto ExtendAndReplaceConstantOp = [&Ctx](VPWidenCastRecipe *ExtA,
VPWidenCastRecipe *&ExtB,
VPValue *&ValB, VPWidenRecipe *Mul) {
- if (!ExtA || ExtB || !isa<VPLiveIn>(ValB))
+ if (!ExtA || ExtB || !isa<VPIRValue>(ValB))
return;
Type *NarrowTy = Ctx.Types.inferScalarType(ExtA->getOperand(0));
Instruction::CastOps ExtOpc = ExtA->getOpcode();
@@ -4207,7 +4207,7 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
auto *VectorPreheader = Plan.getVectorPreheader();
for (VPValue *VPV : VPValues) {
if (vputils::onlyScalarValuesUsed(VPV) ||
- (isa<VPLiveIn>(VPV) && isa<Constant>(VPV->getLiveInIRValue())))
+ (isa<VPIRValue>(VPV) && isa<Constant>(VPV->getLiveInIRValue())))
continue;
// Add explicit broadcast at the insert point that dominates all users.
@@ -4513,7 +4513,7 @@ void VPlanTransforms::materializeConstantVectorTripCount(
if (!Plan.hasScalarTail() ||
Plan.getMiddleBlock()->getSingleSuccessor() ==
Plan.getScalarPreheader() ||
- !isa<VPLiveIn>(TC))
+ !isa<VPIRValue>(TC))
return;
// Materialize vector trip counts for constants early if it can simply
@@ -4833,7 +4833,7 @@ static bool isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
/// Returns true if \p VPValue is a narrow VPValue.
static bool isAlreadyNarrow(VPValue *VPV) {
- if (isa<VPLiveIn>(VPV))
+ if (isa<VPIRValue>(VPV))
return true;
auto *RepR = dyn_cast<VPReplicateRecipe>(VPV);
return RepR && RepR->isSingleScalar();
@@ -5065,7 +5065,7 @@ static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
if (WideIntOrFp && WideIntOrFp->getTruncInst())
return nullptr;
- VPLiveIn *Start = WideIV->getStartValue();
+ VPIRValue *Start = WideIV->getStartValue();
VPValue *Step = WideIV->getStepValue();
const InductionDescriptor &ID = WideIV->getInductionDescriptor();
VPValue *EndValue = VectorTC;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 7d83d77cb57e9..c84dd1459fb06 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -79,7 +79,7 @@ class UnrollState {
void unrollBlock(VPBlockBase *VPB);
VPValue *getValueForPart(VPValue *V, unsigned Part) {
- if (Part == 0 || isa<VPLiveIn, VPSymbolicValue>(V))
+ if (Part == 0 || isa<VPIRValue, VPSymbolicValue>(V))
return V;
assert((VPV2Parts.contains(V) && VPV2Parts[V].size() >= Part) &&
"accessed value does not exist");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 099e79c0fc9b9..d868937e9ea28 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -86,7 +86,7 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
PredicatedScalarEvolution &PSE,
const Loop *L) {
ScalarEvolution &SE = *PSE.getSE();
- if (isa<VPLiveIn, VPSymbolicValue>(V)) {
+ if (isa<VPIRValue, VPSymbolicValue>(V)) {
Value *LiveIn = V->getUnderlyingValue();
if (LiveIn && SE.isSCEVable(LiveIn->getType()))
return SE.getSCEV(LiveIn);
@@ -250,7 +250,7 @@ static bool preservesUniformity(unsigned Opcode) {
bool vputils::isSingleScalar(const VPValue *VPV) {
// A live-in must be uniform across the scope of VPlan.
- if (isa<VPLiveIn, VPSymbolicValue>(VPV))
+ if (isa<VPIRValue, VPSymbolicValue>(VPV))
return true;
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
@@ -288,7 +288,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
// Live-ins are uniform.
- if (isa<VPLiveIn, VPSymbolicValue>(V))
+ if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
VPRecipeBase *R = V->getDefiningRecipe();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 10c7655bb59f3..864f6a30d1357 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -46,9 +46,9 @@ class LLVM_ABI_FOR_TEST VPValue {
friend class VPDef;
friend struct VPDoubleValueDef;
friend class VPlan;
- friend struct VPLiveIn;
+ friend struct VPIRValue;
friend struct VPSymbolicValue;
- friend class VPDefValue;
+ friend class VPRecipeValue;
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
@@ -70,7 +70,7 @@ class LLVM_ABI_FOR_TEST VPValue {
/// Return the underlying Value attached to this VPValue.
Value *getUnderlyingValue() const { return UnderlyingVal; }
- /// Return the underlying IR value for a VPLiveIn.
+ /// Return the underlying IR value for a VPIRValue.
Value *getLiveInIRValue() const;
/// An enumeration for keeping track of the concrete subclass of VPValue that
@@ -178,9 +178,9 @@ LLVM_ABI_FOR_TEST raw_ostream &operator<<(raw_ostream &OS,
/// A VPValue representing a live-in from the input IR. It wraps an underlying
/// IR Value.
-struct VPLiveIn : public VPValue {
- VPLiveIn(Value *UV) : VPValue(VPVLiveInSC, UV, nullptr) {
- assert(UV && "VPLiveIn requires an underlying IR value");
+struct VPIRValue : public VPValue {
+ VPIRValue(Value *UV) : VPValue(VPVLiveInSC, UV, nullptr) {
+ assert(UV && "VPIRValue requires an underlying IR value");
}
/// Returns the underlying IR value.
@@ -205,16 +205,16 @@ struct VPSymbolicValue : public VPValue {
};
/// A VPValue defined by a recipe that produces one or more values.
-class VPDefValue : public VPValue {
+class VPRecipeValue : public VPValue {
friend class VPValue;
friend class VPDef;
/// Pointer to the VPDef that defines this VPValue.
VPDef *Def;
public:
- VPDefValue(VPDef *Def, Value *UV = nullptr);
+ VPRecipeValue(VPDef *Def, Value *UV = nullptr);
- virtual ~VPDefValue();
+ virtual ~VPRecipeValue();
static bool classof(const VPValue *V) {
return V->getVPValueID() == VPVDefValueSC;
@@ -328,16 +328,16 @@ class VPUser {
/// from VPDef before VPValue.
class VPDef {
friend class VPValue;
- friend class VPDefValue;
+ friend class VPRecipeValue;
/// Subclass identifier (for isa/dyn_cast).
const unsigned char SubclassID;
/// The VPValues defined by this VPDef.
- TinyPtrVector<VPDefValue *> DefinedValues;
+ TinyPtrVector<VPRecipeValue *> DefinedValues;
/// Add \p V as a defined value by this VPDef.
- void addDefinedValue(VPDefValue *V) {
+ void addDefinedValue(VPRecipeValue *V) {
assert(V->Def == this &&
"can only add VPValue already linked with this VPDef");
DefinedValues.push_back(V);
@@ -345,7 +345,7 @@ class VPDef {
/// Remove \p V from the values defined by this VPDef. \p V must be a defined
/// value of this VPDef.
- void removeDefinedValue(VPDefValue *V) {
+ void removeDefinedValue(VPRecipeValue *V) {
assert(V->Def == this && "can only remove VPValue linked with this VPDef");
assert(is_contained(DefinedValues, V) &&
"VPValue to remove must be in DefinedValues");
@@ -409,7 +409,7 @@ class VPDef {
VPDef(const unsigned char SC) : SubclassID(SC) {}
virtual ~VPDef() {
- for (VPDefValue *D : make_early_inc_range(DefinedValues)) {
+ for (VPRecipeValue *D : make_early_inc_range(DefinedValues)) {
assert(D->Def == this &&
"all defined VPValues should point to the containing VPDef");
assert(D->getNumUsers() == 0 &&
@@ -443,9 +443,9 @@ class VPDef {
}
/// Returns an ArrayRef of the values defined by the VPDef.
- ArrayRef<VPDefValue *> definedValues() { return DefinedValues; }
+ ArrayRef<VPRecipeValue *> definedValues() { return DefinedValues; }
/// Returns an ArrayRef of the values defined by the VPDef.
- ArrayRef<VPDefValue *> definedValues() const { return DefinedValues; }
+ ArrayRef<VPRecipeValue *> definedValues() const { return DefinedValues; }
/// Returns the number of values defined by the VPDef.
unsigned getNumDefinedValues() const { return DefinedValues.size(); }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
index 582094bed3ef7..e0f7ede5fa912 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanPatternMatchTest.cpp
@@ -27,7 +27,7 @@ TEST_F(VPPatternMatchTest, ScalarIVSteps) {
VPBuilder Builder(VPBB);
IntegerType *I64Ty = IntegerType::get(C, 64);
- VPValue *StartV = Plan.getOrAddLiveIn(ConstantInt::get(I64Ty, 0));
+ VPIRValue *StartV = Plan.getConstantInt(I64Ty, 0);
auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DebugLoc());
Builder.insert(CanonicalIVPHI);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index fb1ffd717ae00..8f9194aa9f099 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1709,8 +1709,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
struct VPDoubleValueDef : public VPRecipeBase {
VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
- new VPDefValue(this);
- new VPDefValue(this);
+ new VPRecipeValue(this);
+ new VPRecipeValue(this);
}
VPRecipeBase *clone() override { return nullptr; }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 91c71724205c9..897c08b66563e 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -21,7 +21,7 @@ using VPVerifierTest = VPlanTestBase;
namespace {
TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
VPlan &Plan = getPlan();
- VPValue *Zero = Plan.getConstantInt(32, 0);
+ VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero, Zero});
VPInstruction *UseI = new VPInstruction(Instruction::Sub, {DefI, Zero});
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
@@ -56,7 +56,7 @@ TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
TEST_F(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
VPlan &Plan = getPlan();
- VPValue *Zero = Plan.getConstantInt(32, 0);
+ VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero, Zero});
VPInstruction *UseI = new VPInstruction(Instruction::Sub, {DefI, Zero});
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
@@ -97,7 +97,7 @@ TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
VPlan &Plan = getPlan();
IntegerType *Int32 = IntegerType::get(C, 32);
auto *Phi = PHINode::Create(Int32, 1);
- VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 0));
+ VPIRValue *Zero = Plan.getConstantInt(Int32, 0);
VPInstruction *DefI = new VPInstruction(Instruction::Add, {Zero, Zero});
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
@@ -146,7 +146,7 @@ TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) {
VPlan &Plan = getPlan();
IntegerType *Int32 = IntegerType::get(C, 32);
- VPValue *Zero = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 0));
+ VPIRValue *Zero = Plan.getConstantInt(Int32, 0);
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
@@ -184,7 +184,7 @@ TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) {
TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) {
VPlan &Plan = getPlan();
- VPValue *Zero = Plan.getConstantInt(32, 0);
+ VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *I1 = new VPInstruction(Instruction::Add, {Zero, Zero});
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
VPInstruction *BranchOnCond =
@@ -218,7 +218,7 @@ TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) {
TEST_F(VPVerifierTest, DuplicateSuccessorsInsideRegion) {
VPlan &Plan = getPlan();
- VPValue *Zero = Plan.getConstantInt(32, 0);
+ VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *I1 = new VPInstruction(Instruction::Add, {Zero, Zero});
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
VPInstruction *BranchOnCond =
@@ -259,7 +259,7 @@ TEST_F(VPVerifierTest, BlockOutsideRegionWithParent) {
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
- VPValue *Zero = Plan.getConstantInt(32, 0);
+ VPIRValue *Zero = Plan.getConstantInt(32, 0);
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
VPBB2->appendRecipe(CanIV);
@@ -288,7 +288,7 @@ TEST_F(VPVerifierTest, BlockOutsideRegionWithParent) {
TEST_F(VPVerifierTest, NonHeaderPHIInHeader) {
VPlan &Plan = getPlan();
- VPValue *Zero = Plan.getConstantInt(32, 0);
+ VPIRValue *Zero = Plan.getConstantInt(32, 0);
auto *CanIV = new VPCanonicalIVPHIRecipe(Zero, {});
auto *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {CanIV});
More information about the llvm-commits
mailing list