[llvm] [VPlan] Assert that replacement types match in VPUser::setOperand (NFC). (PR #195891)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 10:52:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Add assertion to VPValue::setOperand to check if types of the new
operand matches the old operand.
This makes it easier to catch replacements with incorrect types at the
source, instead only later during verification.
A few places currently preform replacements with mis-matching types,
which only get fixed up later. Update those to avoid type-violation.
Depends on https://github.com/llvm/llvm-project/pull/195485 (included in
PR)
---
Patch is 92.92 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/195891.diff
23 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+6-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.cpp (+43-13)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+94-57)
- (modified) llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp (+11-56)
- (modified) llvm/lib/Transforms/Vectorize/VPlanAnalysis.h (+2-10)
- (modified) llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp (+4-3)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+16-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+93-51)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp (+1-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanValue.h (+76-16)
- (modified) llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp (+56)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll (+2-2)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll (+6-6)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll (+5-4)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll (+10-8)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics.ll (+5-4)
- (modified) llvm/unittests/Transforms/Vectorize/VPlanTest.cpp (+4-7)
- (modified) llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp (+38)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ee90c09085c3f..9120409ae60d9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -326,7 +326,7 @@ cl::opt<bool>
#ifdef EXPENSIVE_CHECKS
cl::init(true),
#else
- cl::init(false),
+ cl::init(true),
#endif
cl::Hidden,
cl::desc("Verify VPlans after VPlan transforms."));
@@ -7257,7 +7257,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
ToDelete.push_back(AnyOfSelect);
// Convert the reduction phi to operate on bools.
- PhiR->setOperand(0, Plan->getFalse());
+ auto *NewPhiR =
+ PhiR->cloneWithOperands(Plan->getFalse(), PhiR->getBackedgeValue());
+ NewPhiR->insertBefore(PhiR);
+ PhiR->replaceAllUsesWith(NewPhiR);
+ PhiR = NewPhiR;
// Update NewExitingVPV if it was pointing to the now-replaced select.
if (NewExitingVPV == AnyOfSelect)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 453b5eec9fe5e..fb15c3f78d89c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -122,17 +122,22 @@ void VPRecipeBase::dump() const {
#endif
#if !defined(NDEBUG)
-bool VPRecipeValue::isDefinedBy(const VPDef *D) const { return Def == D; }
+bool VPRecipeValue::isDefinedBy(const VPDef *D) const {
+ return getDefiningRecipe() == D;
+}
#endif
VPRecipeBase *VPValue::getDefiningRecipe() {
auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ if (!DefValue)
+ return nullptr;
+ if (auto *SV = dyn_cast<VPStandaloneRecipeValue>(DefValue))
+ return SV->getDef();
+ return static_cast<VPSingleDefRecipe *>(DefValue);
}
const VPRecipeBase *VPValue::getDefiningRecipe() const {
- auto *DefValue = dyn_cast<VPRecipeValue>(this);
- return DefValue ? DefValue->Def : nullptr;
+ return const_cast<VPValue *>(this)->getDefiningRecipe();
}
Value *VPValue::getLiveInIRValue() const {
@@ -141,15 +146,37 @@ Value *VPValue::getLiveInIRValue() const {
Type *VPIRValue::getType() const { return getUnderlyingValue()->getType(); }
-VPRecipeValue::VPRecipeValue(VPRecipeBase *Def, Value *UV)
- : VPValue(VPVRecipeValueSC, UV), Def(Def) {
- assert(Def && "VPRecipeValue requires a defining recipe");
+Type *VPValue::getScalarType() const {
+ switch (getVPValueID()) {
+ case VPVIRValueSC:
+ return getUnderlyingValue()->getType();
+ case VPVSymbolicSC:
+ return cast<VPSymbolicValue>(this)->getType();
+ case VPRegionValueSC:
+ return cast<VPRegionValue>(this)->getType();
+ case VPVStandaloneRecipeValueSC:
+ case VPVSingleDefValueSC:
+ return cast<VPRecipeValue>(this)->getScalarType();
+ }
+ llvm_unreachable("Unhandled VPValue subclass");
+}
+
+VPSingleDefValue::VPSingleDefValue(VPSingleDefRecipe *Def, Value *UV, Type *Ty)
+ : VPRecipeValue(VPVSingleDefValueSC, UV, Ty) {
+ assert(Def && "VPSingleDefValue requires a defining recipe");
+ Def->addDefinedValue(this);
+}
+
+VPRecipeValue::~VPRecipeValue() = default;
+
+VPStandaloneRecipeValue::VPStandaloneRecipeValue(VPRecipeBase *Def, Value *UV,
+ Type *Ty)
+ : VPRecipeValue(VPVStandaloneRecipeValueSC, UV, Ty), Def(Def) {
+ assert(Def && "VPStandaloneRecipeValue requires a defining recipe");
Def->addDefinedValue(this);
}
-VPRecipeValue::~VPRecipeValue() {
- assert(Users.empty() &&
- "trying to delete a VPRecipeValue with remaining users");
+VPStandaloneRecipeValue::~VPStandaloneRecipeValue() {
Def->removeDefinedValue(this);
}
@@ -862,7 +889,8 @@ VPInstruction *VPRegionBlock::getOrCreateCanonicalIVIncrement() {
CanIV->getDebugLoc(), "index.next");
}
-VPlan::VPlan(Loop *L) {
+VPlan::VPlan(Loop *L, Type *IdxTy)
+ : VectorTripCount(IdxTy), VF(IdxTy), UF(IdxTy), VFxUF(IdxTy) {
setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
ScalarHeader = createVPIRBasicBlock(L->getHeader());
@@ -1228,7 +1256,8 @@ VPlan *VPlan::duplicate() {
NewScalarHeader = createVPIRBasicBlock(ScalarHeaderIRBB);
}
// Create VPlan, clone live-ins and remap operands in the cloned blocks.
- auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
+ auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader,
+ VF.getType());
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
for (VPIRValue *OldLiveIn : getLiveIns())
Old2NewVPValues[OldLiveIn] = NewPlan->getOrAddLiveIn(OldLiveIn);
@@ -1250,7 +1279,8 @@ VPlan *VPlan::duplicate() {
"All VPSymbolicValues must be handled below");
if (BackedgeTakenCount)
- NewPlan->BackedgeTakenCount = new VPSymbolicValue();
+ NewPlan->BackedgeTakenCount =
+ new VPSymbolicValue(BackedgeTakenCount->getType());
// Map and propagate materialized state for symbolic values.
for (auto [OldSV, NewSV] :
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a1ea6b3439bf..1e5c57874f477 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -603,18 +603,29 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
return R->getVPRecipeID() == VPRecipeID; \
}
+/// Return the scalar type of \p V. If \p V's scalar type has not been set
+/// (e.g. the defining recipe has not yet been migrated), fall back to
+/// VPTypeAnalysis using the plan of the defining recipe.
+LLVM_ABI Type *getScalarTypeOrInfer(VPValue *V);
+
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
/// more output IR that define a single result VPValue.
-/// Note that VPRecipeBase must be inherited from before VPValue.
-class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
+class VPSingleDefRecipe : public VPRecipeBase, public VPSingleDefValue {
public:
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this) {}
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV) {}
+
+ VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Type *Ty, Value *UV = nullptr,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeBase(SC, Operands, DL), VPSingleDefValue(this, UV, Ty) {}
+
+ ~VPSingleDefRecipe() override { removeDefinedValue(this); }
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -1116,6 +1127,12 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
DebugLoc DL = DebugLoc::getUnknown())
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
+ VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, const VPIRFlags &Flags,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPSingleDefRecipe(SC, Operands, ResultTy, /*UV=*/nullptr, DL),
+ VPIRFlags(Flags) {}
+
static inline bool classof(const VPRecipeBase *R) {
return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
R->getVPRecipeID() == VPRecipeBase::VPInstructionSC ||
@@ -1797,11 +1814,13 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
~VPWidenRecipe() override = default;
- VPWidenRecipe *clone() override {
+ VPWidenRecipe *clone() override { return cloneWithOperands(operands()); }
+
+ VPWidenRecipe *cloneWithOperands(ArrayRef<VPValue *> NewOperands) {
if (auto *UV = getUnderlyingValue())
- return new VPWidenRecipe(*cast<Instruction>(UV), operands(), *this, *this,
- getDebugLoc());
- return new VPWidenRecipe(Opcode, operands(), *this, *this, getDebugLoc());
+ return new VPWidenRecipe(*cast<Instruction>(UV), NewOperands, *this,
+ *this, getDebugLoc());
+ return new VPWidenRecipe(Opcode, NewOperands, *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC)
@@ -1837,16 +1856,14 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// Cast instruction opcode.
Instruction::CastOps Opcode;
- /// Result type for the cast.
- Type *ResultTy;
-
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, ResultTy, Flags,
+ DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
assert(hasRequiredFlagsForOpcode(Opcode) &&
@@ -1857,7 +1874,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
+ return new VPWidenCastRecipe(Opcode, getOperand(0), getScalarType(),
cast_or_null<CastInst>(getUnderlyingValue()),
*this, *this, getDebugLoc());
}
@@ -1874,7 +1891,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
Instruction::CastOps getOpcode() const { return Opcode; }
/// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
+ Type *getResultType() const { return getScalarType(); }
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1889,9 +1906,6 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
/// ID of the vector intrinsic to widen.
Intrinsic::ID VectorIntrinsicID;
- /// Scalar return type of the intrinsic.
- Type *ResultTy;
-
/// True if the intrinsic may read from memory.
bool MayReadFromMemory;
@@ -1907,9 +1921,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments, Ty,
Flags, DL),
- VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {
@@ -1921,10 +1935,9 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments, Ty,
Flags, DL),
- VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
- ResultTy(Ty) {
+ VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID) {
LLVMContext &Ctx = Ty->getContext();
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
MemoryEffects ME = Attrs.getMemoryEffects();
@@ -1940,10 +1953,11 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
- operands(), ResultTy, *this, *this,
- getDebugLoc());
- return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
- *this, *this, getDebugLoc());
+ operands(), getResultType(), *this,
+ *this, getDebugLoc());
+ return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(),
+ getResultType(), *this, *this,
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntrinsicSC)
@@ -1959,7 +1973,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
/// Return the scalar return type of the intrinsic.
- Type *getResultType() const { return ResultTy; }
+ Type *getResultType() const { return getScalarType(); }
/// Return to name of the intrinsic as string.
StringRef getIntrinsicName() const;
@@ -1996,8 +2010,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments,
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, CallArguments, Flags,
- DL),
+ : VPRecipeWithIRFlags(
+ VPRecipeBase::VPWidenCallSC, CallArguments,
+ cast<Function>(CallArguments.back()->getLiveInIRValue())
+ ->getReturnType(),
+ Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
@@ -2098,7 +2115,8 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands,
+ getScalarTypeOrInfer(Operands[0]), Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
@@ -2158,7 +2176,7 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *SourceElementTy,
int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
: VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC, {Ptr, VF},
- GEPFlags, DL),
+ getScalarTypeOrInfer(Ptr), GEPFlags, DL),
SourceElementTy(SourceElementTy), Stride(Stride) {
assert(Stride < 0 && "Stride must be negative");
}
@@ -2226,7 +2244,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr,
+ getScalarTypeOrInfer(Ptr), GEPFlags, DL),
SourceElementTy(SourceElementTy) {}
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
@@ -2477,10 +2496,15 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
~VPWidenIntOrFpInductionRecipe() override = default;
+ VPWidenIntOrFpInductionRecipe *cloneWithOperands(VPIRValue *Start,
+ VPValue *Step, VPValue *VF) {
+ return new VPWidenIntOrFpInductionRecipe(getPHINode(), Start, Step, VF,
+ getInductionDescriptor(), Trunc,
+ *this, getDebugLoc());
+ }
+
VPWidenIntOrFpInductionRecipe *clone() override {
- return new VPWidenIntOrFpInductionRecipe(
- getPHINode(), getStartValue(), getStepValue(), getVFValue(),
- getInductionDescriptor(), Trunc, *this, getDebugLoc());
+ return cloneWithOperands(getStartValue(), getStepValue(), getVFValue());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntOrFpInductionSC)
@@ -2585,11 +2609,13 @@ class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
std::string Name;
public:
- /// Create a new VPWidenPHIRecipe with incoming values \p IncomingvValues,
+ /// Create a new VPWidenPHIRecipe with incoming values \p IncomingValues,
/// debug location \p DL and \p Name.
VPWidenPHIRecipe(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues, DL),
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues,
+ getScalarTypeOrInfer(IncomingValues[0]),
+ /*UV=*/nullptr, DL),
Name(Name.str()) {}
VPWidenPHIRecipe *clone() override {
@@ -2708,11 +2734,15 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPIRFlags {
~VPReductionPHIRecipe() override = default;
- VPReductionPHIRecipe *clone() override {
+ VPReductionPHIRecipe *cloneWithOperands(VPValue *Start,
+ VPValue *BackedgeValue) {
return new VPReductionPHIRecipe(
dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
- *getOperand(0), *getBackedgeValue(), Style, *this,
- HasUsesOutsideReductionChain);
+ *Start, *BackedgeValue, Style, *this, HasUsesOutsideReductionChain);
+ }
+
+ VPReductionPHIRecipe *clone() override {
+ return cloneWithOperands(getOperand(0), getBackedgeValue());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPReductionPHISC)
@@ -2873,7 +2903,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
if (StoredValues.empty()) {
for (Instruction *Inst : IG->members()) {
assert(!Inst->getType()->isVoidTy() && "must have result");
- new VPRecipeValue(this, Inst);
+ new VPStandaloneRecipeValue(this, Inst, Inst->getType());
}
} else {
for (auto *SV : StoredValues)
@@ -3459,7 +3489,8 @@ class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
- : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV, DL) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV,
+ getScalarTypeOrInfer(PredV), nullptr, DL) {}
~VPPredInstPHIRecipe() override = default;
VPPredInstPHIRecipe *clone() override {
@@ -3575,13 +3606,14 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
/// A recipe for widening load operations, using the address to load from and an
/// optional mask.
-struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
- public VPRecipeValue {
+struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final
+ : public VPWidenMemoryRecipe,
+ public VPStandaloneRecipeValue {
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
Consecuti...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/195891
More information about the llvm-commits
mailing list