[llvm] [VPlan] Remove type verification from VPlanVerifier (NFC) (PR #201209)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 11:56:06 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/201209
>From 68a3f370392430f73659cfffaac00d0ff0014a07 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 31 May 2026 14:43:07 +0100
Subject: [PATCH 1/2] [VPlan] Restrict addOperand to sub-classes that need it
(NFC).
Make addOperand protected and add dedicated helpers in sub-classes that
verify the type of the added operand if possible.
Most recipes already add all their operands on construction. This patch
makes sure that addOperand cannot be used to add operands with invalid
types.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 74 +++++++++++++++++--
.../Vectorize/VPlanConstruction.cpp | 12 +--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 43 ++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +--
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 27 ++++---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 12 +--
.../Transforms/Vectorize/VPlanTest.cpp | 43 +++++++----
.../Vectorize/VPlanVerifierTest.cpp | 6 +-
8 files changed, 177 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ef64e30ac38e6..40261ed1c2c0f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1415,6 +1415,11 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
unsigned getOpcode() const { return Opcode; }
+ /// Add \p Op as operand of this VPInstruction. Only supported for AnyOf,
+ /// ComputeReductionResult, BuildVector, BuildStructVector, ExtractLane,
+ /// ExtractLastActive, FirstActiveLane, LastActiveLane.
+ void addOperand(VPValue *Op);
+
/// Generate the instruction.
/// TODO: We currently execute only per-part unless a specific instance is
/// provided.
@@ -1473,7 +1478,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
assert(!isMasked() && "recipe is already masked");
if (alwaysUnmasked())
return;
- addOperand(Mask);
+ assert(Mask->getScalarType()->isIntegerTy(1) &&
+ "Mask must be an i1 (vector)");
+ VPUser::addOperand(Mask);
}
/// Returns the mask for the VPInstruction. Returns nullptr for unmasked
@@ -1647,6 +1654,15 @@ class VPPhiAccessors {
/// predecessor.
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const;
+ /// Append \p IncomingV as an incoming value to the phi-like recipe.
+ void addIncoming(VPValue *IncomingV) {
+ auto *R = const_cast<VPRecipeBase *>(getAsRecipe());
+ assert((R->getNumOperands() == 0 ||
+ IncomingV->getScalarType() == R->getOperand(0)->getScalarType()) &&
+ "all incoming values must have the same type");
+ R->addOperand(IncomingV);
+ }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const;
@@ -2278,6 +2294,14 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
/// Offset = Stride * (VF - 1) + Part * Stride * VF.
void materializeOffset(unsigned Part = 0);
+ /// Append \p Offset as the offset operand. The offset is an integer index
+ /// expressed in units of SourceElementTy.
+ void addOffset(VPValue *Offset) {
+ assert(Offset->getScalarType()->isIntegerTy() &&
+ "offset must be an integer index");
+ VPUser::addOperand(Offset);
+ }
+
void execute(VPTransformState &State) override;
bool usesFirstLaneOnly(const VPValue *Op) const override {
@@ -2306,7 +2330,7 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
getPointer(), getVFValue(), getSourceElementType(), getStride(),
getGEPNoWrapFlags(), getDebugLoc());
if (auto *Offset = getOffset())
- VEPR->addOperand(Offset);
+ VEPR->addOffset(Offset);
return VEPR;
}
@@ -2341,6 +2365,13 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
return getNumOperands() > 2 ? getOperand(2) : nullptr;
}
+ /// Add the per-part offset (VFxPart) used for unrolled parts > 0.
+ void addPerPartOffset(VPValue *VFxPart) {
+ assert(VFxPart->getScalarType()->isIntegerTy() &&
+ "per-part offset must be an integer index");
+ VPUser::addOperand(VFxPart);
+ }
+
void execute(VPTransformState &State) override;
Type *getSourceElementType() const { return SourceElementTy; }
@@ -2364,7 +2395,7 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
new VPVectorPointerRecipe(getOperand(0), SourceElementTy, getStride(),
getGEPNoWrapFlags(), getDebugLoc());
if (auto *VFxPart = getVFxPart())
- Clone->addOperand(VFxPart);
+ Clone->addPerPartOffset(VFxPart);
return Clone;
}
@@ -2457,6 +2488,15 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
/// Update the incoming value from the loop backedge.
void setBackedgeValue(VPValue *V) { setOperand(1, V); }
+ /// Add \p V as the incoming value from the loop backedge.
+ void addBackedgeValue(VPValue *V) {
+ assert(getNumOperands() == 1 &&
+ "backedge value must be appended right after construction");
+ assert(V->getScalarType() == getScalarType() &&
+ "backedge value must have the same type as the start value");
+ VPUser::addOperand(V);
+ }
+
/// Returns the backedge value as a recipe. The backedge value is guaranteed
/// to be a recipe.
virtual VPRecipeBase &getBackedgeRecipe() {
@@ -2491,6 +2531,20 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
addOperand(Step);
}
+ /// After unrolling, append the splat-VF step (`VF * step`) and the value of
+ /// the induction at the last unrolled part.
+ void addUnrolledPartOperands(VPValue *SplatVFStep, VPValue *LastPart) {
+ assert(LastPart->getScalarType() == getScalarType() &&
+ "last-part value must match the induction recipe's scalar type");
+ assert((getScalarType()->isPointerTy()
+ ? SplatVFStep->getScalarType()->isIntegerTy()
+ : SplatVFStep->getScalarType() == getScalarType()) &&
+ "splat-step must match the induction type for non-pointer "
+ "inductions, or be an integer index for pointer inductions");
+ VPUser::addOperand(SplatVFStep);
+ VPUser::addOperand(LastPart);
+ }
+
static inline bool classof(const VPRecipeBase *R) {
return R->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC ||
R->getVPRecipeID() == VPRecipeBase::VPWidenPointerInductionSC;
@@ -2964,6 +3018,7 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
/// Set mask number \p Idx to \p V.
void setMask(unsigned Idx, VPValue *V) {
assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
+ assert(V->getScalarType()->isIntegerTy(1) && "Mask must be an i1 (vector)");
Idx == 0 ? setOperand(1, V) : setOperand(Idx * 2 + !isNormalized(), V);
}
@@ -3683,6 +3738,8 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
assert(!IsMasked && "cannot re-set mask");
if (!Mask)
return;
+ assert(Mask->getScalarType()->isIntegerTy(1) &&
+ "Mask must be an i1 (vector)");
getAsRecipe()->addOperand(Mask);
IsMasked = true;
}
@@ -3979,7 +4036,7 @@ class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
VPActiveLaneMaskPHIRecipe *clone() override {
auto *R = new VPActiveLaneMaskPHIRecipe(getOperand(0), getDebugLoc());
if (getNumOperands() == 2)
- R->addOperand(getOperand(1));
+ R->addBackedgeValue(getOperand(1));
return R;
}
@@ -4057,7 +4114,7 @@ class VPWidenCanonicalIVRecipe : public VPRecipeWithIRFlags {
auto *WideCanIV =
new VPWidenCanonicalIVRecipe(getCanonicalIV(), getNoWrapFlags());
if (VPValue *Step = getStepValue())
- WideCanIV->addOperand(Step);
+ WideCanIV->addPerPartStep(Step);
return WideCanIV;
}
@@ -4083,6 +4140,13 @@ class VPWidenCanonicalIVRecipe : public VPRecipeWithIRFlags {
return getNumOperands() == 2 ? getOperand(1) : nullptr;
}
+ /// Add the per-part step (VF * Part) used for unrolled parts.
+ void addPerPartStep(VPValue *Step) {
+ assert(Step->getScalarType() == getScalarType() &&
+ "per-part step must have the same type as the canonical IV");
+ VPUser::addOperand(Step);
+ }
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index e5c546c7fb851..6f087c4ce801c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -117,7 +117,7 @@ void PlainCFGBuilder::fixHeaderPhis() {
assert(Phi->getNumOperands() == 2 &&
"header phi must have exactly 2 operands");
for (BasicBlock *Pred : predecessors(Phi->getParent()))
- PhiR->addOperand(
+ PhiR->addIncoming(
getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
}
}
@@ -244,7 +244,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
getOrCreateVPOperand(Phi->getIncomingValue(I));
}
for (VPBlockBase *Pred : VPBB->getPredecessors())
- NewR->addOperand(
+ cast<VPPhi>(NewR)->addIncoming(
VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock()));
}
} else {
@@ -371,7 +371,7 @@ std::unique_ptr<VPlan> PlainCFGBuilder::buildPlainCFG() {
assert(PhiR->getNumOperands() == 0 &&
"no phi operands should be added yet");
for (BasicBlock *Pred : predecessors(EB->getIRBasicBlock()))
- PhiR->addOperand(
+ PhiR->addIncoming(
getOrCreateVPOperand(Phi.getIncomingValueForBlock(Pred)));
}
}
@@ -591,7 +591,7 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
{ResumeFromVectorLoop, VectorPhiR->getOperand(0)},
VectorPhiR->getDebugLoc());
- cast<VPIRPhi>(&ScalarPhiR)->addOperand(ResumePhiR);
+ cast<VPIRPhi>(&ScalarPhiR)->addIncoming(ResumePhiR);
}
}
@@ -1456,7 +1456,7 @@ static void insertCheckBlockBeforeVectorLoop(VPlan &Plan,
auto *Phi = cast<VPPhi>(&R);
assert(Phi->getNumIncoming() == NumPreds - 1 &&
"must have incoming values for all predecessors");
- Phi->addOperand(Phi->getOperand(NumPreds - 2));
+ Phi->addIncoming(Phi->getOperand(NumPreds - 2));
}
}
@@ -1889,7 +1889,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
VPValue *AnyOf = Builder.createNaryOp(VPInstruction::AnyOf, {Cond});
VPValue *MaskSelect = Builder.createSelect(AnyOf, Cond, MaskPHI);
- MaskPHI->addOperand(MaskSelect);
+ MaskPHI->addIncoming(MaskSelect);
// Replace select for data.
VPValue *DataSelect =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 941e8e309471e..28e4b0ca68e1f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1443,6 +1443,47 @@ bool VPInstruction::isSingleScalar() const {
}
}
+void VPInstruction::addOperand(VPValue *Op) {
+#ifndef NDEBUG
+ Type *Ty = Op->getScalarType();
+ switch (getOpcode()) {
+ case VPInstruction::AnyOf:
+ case VPInstruction::FirstActiveLane:
+ case VPInstruction::LastActiveLane:
+ assert(Ty == getOperand(0)->getScalarType() && Ty->isIntegerTy(1) &&
+ "appended operand must be an i1 (vector) matching operand 0");
+ break;
+ case VPInstruction::ComputeReductionResult:
+ case VPInstruction::BuildVector:
+ case VPInstruction::BuildStructVector:
+ assert(Ty == getOperand(0)->getScalarType() &&
+ "appended operand must match operand 0's scalar type");
+ break;
+ case VPInstruction::ExtractLane:
+ assert(Ty == getOperand(1)->getScalarType() &&
+ "appended operand must match operand 1's scalar type");
+ break;
+ case VPInstruction::ExtractLastActive: {
+ // The recipe is constructed with 3 operands (result, data, mask). Extra
+ // operands beyond that are appended in (data, mask) pairs.
+ constexpr unsigned NumInitialOperands = 3;
+ assert(getNumOperands() >= NumInitialOperands &&
+ "ExtractLastActive must have at least the initial 3 operands");
+ bool IsMaskSlot = ((getNumOperands() - NumInitialOperands) & 1u) == 1u;
+ assert((IsMaskSlot ? Ty->isIntegerTy(1)
+ : Ty == getOperand(1)->getScalarType()) &&
+ "ExtractLastActive expects alternating data/mask operands "
+ "matching operand 1's type and i1, respectively");
+ break;
+ }
+ default:
+ llvm_unreachable("opcode does not support growing the operand list "
+ "outside of construction");
+ }
+#endif
+ VPUser::addOperand(Op);
+}
+
void VPInstruction::execute(VPTransformState &State) {
assert(!isMasked() && "cannot execute masked VPInstruction");
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
@@ -2954,7 +2995,7 @@ void VPVectorEndPointerRecipe::materializeOffset(unsigned Part) {
VPValue *Offset = Builder.createAdd(
Offset0,
Builder.createOverflowingOp(Instruction::Mul, {PartxStride, VF}));
- addOperand(Offset);
+ addOffset(Offset);
}
void VPVectorEndPointerRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6a45a9a95ea94..eca8cddb0e40d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2883,7 +2883,7 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
auto *ALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
{InLoopIncrement, TC, ALMMultiplier}, DL,
"active.lane.mask.next");
- LaneMaskPhi->addOperand(ALM);
+ LaneMaskPhi->addBackedgeValue(ALM);
// Replace the original terminator with BranchOnCond. We have to invert the
// mask here because a true condition means jumping to the exit block.
@@ -3348,12 +3348,12 @@ void VPlanTransforms::addExplicitVectorLength(
auto *NextIter = Builder.createAdd(
OpVPEVL, CurrentIteration, CanonicalIVIncrement->getDebugLoc(),
"current.iteration.next", CanonicalIVIncrement->getNoWrapFlags());
- CurrentIteration->addOperand(NextIter);
+ CurrentIteration->addBackedgeValue(NextIter);
VPValue *NextAVL =
Builder.createSub(AVLPhi, OpVPEVL, DebugLoc::getCompilerGenerated(),
"avl.next", {/*NUW=*/true, /*NSW=*/false});
- AVLPhi->addOperand(NextAVL);
+ AVLPhi->addIncoming(NextAVL);
fixupVFUsersForEVL(Plan, *VPEVL);
removeDeadRecipes(Plan);
@@ -3834,7 +3834,7 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR) {
auto *Next = Builder.createNaryOp(AddOp, {Prev, Inc}, Flags,
WidenIVR->getDebugLoc(), "vec.ind.next");
- WidePHI->addOperand(Next);
+ WidePHI->addIncoming(Next);
WidenIVR->replaceAllUsesWith(WidePHI);
}
@@ -3897,7 +3897,7 @@ static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R) {
VPValue *InductionGEP =
Builder.createPtrAdd(ScalarPtrPhi, Inc, DL, "ptr.ind");
- ScalarPtrPhi->addOperand(InductionGEP);
+ ScalarPtrPhi->addIncoming(InductionGEP);
}
/// Expand a VPDerivedIVRecipe into executable recipes.
@@ -4344,7 +4344,7 @@ void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
DebugLoc::getUnknown(), "early.exit.value");
}
ExitIRI->removeIncomingValueFor(EarlyExitingVPBB);
- ExitIRI->addOperand(NewIncoming);
+ ExitIRI->addIncoming(NewIncoming);
}
EarlyExitingVPBB->getTerminator()->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 82cc5b05ebfaf..bb49059e731d7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -236,8 +236,7 @@ void UnrollState::unrollWidenInductionByUF(
addRecipeForPart(IV, Add, Part);
Prev = Add;
}
- IV->addOperand(VectorStep);
- IV->addOperand(Prev);
+ IV->addUnrolledPartOperands(VectorStep, Prev);
}
void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
@@ -349,7 +348,10 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
VPValue *VFxPart = Builder.createOverflowingOp(
Instruction::Mul, {VF, Plan.getConstantInt(IndexTy, Part)},
{true, true});
- Copy->addOperand(VFxPart);
+ if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Copy))
+ VecPtr->addPerPartOffset(VFxPart);
+ else
+ cast<VPWidenCanonicalIVRecipe>(Copy)->addPerPartStep(VFxPart);
continue;
}
if (auto *Red = dyn_cast<VPReductionRecipe>(&R)) {
@@ -391,7 +393,7 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
}
if (auto *WideCanIV = dyn_cast<VPWidenCanonicalIVRecipe>(&R)) {
// Set Part0 step for WidenCanonicalIV.
- WideCanIV->addOperand(getConstantInt(0));
+ WideCanIV->addPerPartStep(getConstantInt(0));
}
}
@@ -425,26 +427,29 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
match(&R, m_FirstActiveLane(m_VPValue(Op1))) ||
match(&R, m_LastActiveLane(m_VPValue(Op1))) ||
match(&R, m_ComputeReductionResult(m_VPValue(Op1)))) {
- addUniformForAllParts(cast<VPInstruction>(&R));
+ auto *VPI = cast<VPInstruction>(&R);
+ addUniformForAllParts(VPI);
for (unsigned Part = 1; Part != UF; ++Part)
- R.addOperand(getValueForPart(Op1, Part));
+ VPI->addOperand(getValueForPart(Op1, Part));
continue;
}
VPValue *Op0;
if (match(&R, m_ExtractLane(m_VPValue(Op0), m_VPValue(Op1)))) {
- addUniformForAllParts(cast<VPInstruction>(&R));
+ auto *VPI = cast<VPInstruction>(&R);
+ addUniformForAllParts(VPI);
for (unsigned Part = 1; Part != UF; ++Part)
- R.addOperand(getValueForPart(Op1, Part));
+ VPI->addOperand(getValueForPart(Op1, Part));
continue;
}
VPValue *Op2;
if (match(&R, m_ExtractLastActive(m_VPValue(), m_VPValue(Op1),
m_VPValue(Op2)))) {
- addUniformForAllParts(cast<VPInstruction>(&R));
+ auto *VPI = cast<VPInstruction>(&R);
+ addUniformForAllParts(VPI);
for (unsigned Part = 1; Part != UF; ++Part) {
- R.addOperand(getValueForPart(Op1, Part));
- R.addOperand(getValueForPart(Op2, Part));
+ VPI->addOperand(getValueForPart(Op1, Part));
+ VPI->addOperand(getValueForPart(Op2, Part));
}
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 427bc2ea199dc..61c68d4577269 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -384,6 +384,8 @@ class VPMultiDefValue : public VPRecipeValue {
class VPUser {
/// Grant access to removeOperand for VPPhiAccessors, the only supported user.
friend class VPPhiAccessors;
+ /// Grant access to addOperand for VPWidenMemoryRecipe.
+ friend class VPWidenMemoryRecipe;
SmallVector<VPValue *, 2> Operands;
@@ -405,6 +407,11 @@ class VPUser {
addOperand(Operand);
}
+ void addOperand(VPValue *Operand) {
+ Operands.push_back(Operand);
+ Operand->addUser(*this);
+ }
+
public:
VPUser() = delete;
VPUser(const VPUser &) = delete;
@@ -414,11 +421,6 @@ class VPUser {
Op->removeUser(*this);
}
- void addOperand(VPValue *Operand) {
- Operands.push_back(Operand);
- Operand->addUser(*this);
- }
-
unsigned getNumOperands() const { return Operands.size(); }
inline VPValue *getOperand(unsigned N) const {
assert(N < Operands.size() && "Operand index out of bounds");
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index f36f361bbf71d..9937abd60fe1b 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1272,7 +1272,7 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPRecipeBase) {
auto *Load =
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(C));
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, {}, {});
checkVPRecipeCastImpl<VPWidenLoadRecipe, VPUser, VPIRMetadata>(&Recipe);
@@ -1303,7 +1303,7 @@ TEST_F(VPRecipeTest, CastVPWidenLoadEVLRecipeToVPUser) {
auto *Load =
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(C));
VPValue *EVL = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 8));
VPWidenLoadRecipe BaseLoad(*Load, Addr, Mask, true, {}, {});
VPWidenLoadEVLRecipe Recipe(BaseLoad, Addr, *EVL, Mask);
@@ -1321,7 +1321,7 @@ TEST_F(VPRecipeTest, CastVPWidenStoreRecipeToVPUser) {
PoisonValue::get(Int32Ptr), false, Align(1));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *StoredVal = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 42));
- VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(C));
VPWidenStoreRecipe Recipe(*Store, Addr, StoredVal, Mask, true, {}, {});
checkVPRecipeCastImpl<VPWidenStoreRecipe, VPUser, VPIRMetadata>(&Recipe);
@@ -1338,7 +1338,7 @@ TEST_F(VPRecipeTest, CastVPWidenStoreEVLRecipeToVPUser) {
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPValue *StoredVal = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 42));
VPValue *EVL = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 8));
- VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(C));
VPWidenStoreRecipe BaseStore(*Store, Addr, StoredVal, Mask, true, {}, {});
VPWidenStoreEVLRecipe Recipe(BaseStore, Addr, StoredVal, *EVL, Mask);
@@ -1423,7 +1423,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
{
auto *Load =
new LoadInst(Int32, PoisonValue::get(Int32Ptr), "", false, Align(1));
- VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(C));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, {}, {});
EXPECT_FALSE(Recipe.mayHaveSideEffects());
@@ -1436,7 +1436,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
{
auto *Store = new StoreInst(PoisonValue::get(Int32),
PoisonValue::get(Int32Ptr), false, Align(1));
- VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(C));
VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
VPValue *StoredV = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
VPWidenStoreRecipe Recipe(*Store, Addr, StoredV, Mask, false, {}, {});
@@ -1845,11 +1845,10 @@ TEST_F(VPInstructionTest, VPSymbolicValueMaterialization) {
// Create a recipe that uses VF.
VPValue *VF = &Plan.getVF();
- VPInstruction *I = new VPInstructionWithType(VPInstruction::StepVector, {},
- IntegerType::get(C, 32));
+ VPInstruction *I =
+ new VPInstruction(VPInstruction::ExplicitVectorLength, {VF});
VPBasicBlock &VPBB = *Plan.createVPBasicBlock("");
VPBB.appendRecipe(I);
- I->addOperand(VF);
// Replace VF with a constant.
VF->replaceAllUsesWith(Plan.getConstantInt(64, 1));
@@ -1899,7 +1898,7 @@ TEST_F(VPUtilsTest, IsUniformAcrossVFsAndUFsForSingleScalarOpcodes) {
}
#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG)
-TEST_F(VPInstructionTest, VPSymbolicValueAddUserAfterMaterialization) {
+TEST_F(VPInstructionTest, VPSymbolicValueConstructUserAfterMaterialization) {
VPlan &Plan = getPlan();
// Materialize VF by replacing all uses.
@@ -1908,11 +1907,25 @@ TEST_F(VPInstructionTest, VPSymbolicValueAddUserAfterMaterialization) {
EXPECT_TRUE(Plan.getVF().isMaterialized());
// Adding a new user to a materialized value should crash.
- VPInstruction *I = new VPInstructionWithType(VPInstruction::StepVector, {},
- IntegerType::get(C, 32));
- VPBasicBlock &VPBB = *Plan.createVPBasicBlock("");
- VPBB.appendRecipe(I);
- EXPECT_DEATH(I->addOperand(VF), "accessing materialized symbolic value");
+ EXPECT_DEATH(new VPInstruction(VPInstruction::ExplicitVectorLength, {VF}),
+ "accessing materialized symbolic value");
+}
+
+TEST_F(VPInstructionTest, VPSymbolicValueAddOperandAfterMaterialization) {
+ VPlan &Plan = getPlan();
+
+ IntegerType *I64 = IntegerType::get(C, 64);
+ VPValue *Seed = Plan.getOrAddLiveIn(ConstantInt::get(I64, 0));
+ std::unique_ptr<VPInstruction> BV(
+ new VPInstruction(VPInstruction::BuildVector, {Seed}));
+
+ // Materialize VF by replacing all uses.
+ VPValue *VF = &Plan.getVF();
+ VF->replaceAllUsesWith(Plan.getConstantInt(64, 1));
+ EXPECT_TRUE(Plan.getVF().isMaterialized());
+
+ // Appending a new operand referencing the materialized VF should crash.
+ EXPECT_DEATH(BV->addOperand(VF), "accessing materialized symbolic value");
}
#endif
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index de084ec9a070a..638a875df86ea 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -374,8 +374,8 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) {
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
auto Plan = buildVPlan(LoopHeader);
- VPValue *Zero = Plan->getConstantInt(32, 0);
- Plan->getScalarHeader()->front().addOperand(Zero);
+ VPValue *Zero = Plan->getConstantInt(64, 0);
+ cast<VPIRPhi>(&Plan->getScalarHeader()->front())->addIncoming(Zero);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -416,7 +416,7 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) {
new VPInstruction(VPInstruction::ExtractLastLane,
{HeaderBlock->front().getVPSingleValue()});
DefI->insertBefore(Plan->getMiddleBlock()->getTerminator());
- Plan->getExitBlocks()[0]->front().addOperand(DefI);
+ cast<VPIRPhi>(&Plan->getExitBlocks()[0]->front())->addIncoming(DefI);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
>From 31bcac6f0d7bd537f57264a875fc6d9d191f60ff Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 2 Jun 2026 22:32:11 +0100
Subject: [PATCH 2/2] [VPlan] Remove type verification from VPlanVerifier (NFC)
With https://github.com/llvm/llvm-project/pull/200692 (still pending)
and https://github.com/llvm/llvm-project/pull/200686, types are no
checked at construction, and each operation that changes operands
(setOperand, addOperand etc) verifies that the replacement happens with
suitable types.
This should remove the need for running type checking as part of the
verifier.
---
.../Transforms/Vectorize/VPlanVerifier.cpp | 99 -------------------
1 file changed, 99 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index be7f84f3b5a00..900c760dbe6c8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -20,7 +20,6 @@
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/TypeSwitch.h"
#define DEBUG_TYPE "loop-vectorize"
@@ -41,12 +40,6 @@ class VPlanVerifier {
/// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask.
bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const;
- /// Verify that the stored scalar type of \p R is consistent with the types
- /// derived from its operands. A null stored type is tolerated during the
- /// transition to fully threaded scalar types; once set, it must agree with
- /// the operand-derived type.
- bool verifyRecipeTypes(const VPRecipeBase &R) const;
-
bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
bool verifyBlock(const VPBlockBase *VPB);
@@ -218,96 +211,6 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
return true;
}
-bool VPlanVerifier::verifyRecipeTypes(const VPRecipeBase &R) const {
- const auto *SR = dyn_cast<VPSingleDefRecipe>(&R);
- if (!SR)
- return true;
-
- auto CheckScalarType = [&](Type *Derived) -> bool {
- if (Derived == SR->getScalarType())
- return true;
- errs() << "Recipe result type does not match type derived from operands";
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- errs() << ": ";
- R.dump();
-#endif
- errs() << "\n";
- return false;
- };
-
- auto CheckOperandTypes = [&]() -> bool {
- if (all_of(drop_begin(R.operands()), [&R](VPValue *Op) {
- return R.getOperand(0)->getScalarType() == Op->getScalarType();
- }))
- return true;
- errs() << "Recipe operand types do not match";
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- errs() << ": ";
- R.dump();
-#endif
- errs() << "\n";
- return false;
- };
-
- if (auto *WII = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
- return CheckScalarType(WII->getTruncInst()
- ? WII->getTruncInst()->getType()
- : WII->getStartValue()->getScalarType());
-
- switch (R.getVPRecipeID()) {
- case VPRecipeBase::VPVectorPointerSC:
- case VPRecipeBase::VPVectorEndPointerSC:
- case VPRecipeBase::VPWidenGEPSC:
- case VPRecipeBase::VPScalarIVStepsSC:
- case VPRecipeBase::VPWidenPointerInductionSC:
- case VPRecipeBase::VPDerivedIVSC:
- return CheckScalarType(R.getOperand(0)->getScalarType());
- case VPRecipeBase::VPWidenPHISC:
- case VPRecipeBase::VPPredInstPHISC:
- case VPRecipeBase::VPReductionPHISC:
- case VPRecipeBase::VPActiveLaneMaskPHISC:
- case VPRecipeBase::VPCurrentIterationPHISC:
- case VPRecipeBase::VPFirstOrderRecurrencePHISC:
- return CheckOperandTypes() &&
- CheckScalarType(R.getOperand(0)->getScalarType());
- case VPRecipeBase::VPInstructionSC: {
- auto *VPI = cast<VPInstruction>(&R);
- if (isa<VPInstructionWithType>(VPI) ||
- is_contained(
- ArrayRef<unsigned>{
- Instruction::ExtractValue, VPInstruction::FirstActiveLane,
- VPInstruction::LastActiveLane, VPInstruction::NumActiveLanes,
- VPInstruction::IncomingAliasMask, Instruction::Load,
- Instruction::Alloca, Instruction::Call},
- VPI->getOpcode()))
- return true;
- SmallVector<VPValue *, 4> Ops(VPI->operandsWithoutMask());
- return CheckScalarType(
- computeScalarTypeForInstruction(VPI->getOpcode(), Ops));
- }
- case VPRecipeBase::VPReplicateSC: {
- auto *RepR = cast<VPReplicateRecipe>(&R);
- SmallVector<VPValue *, 4> Ops(RepR->operands());
- if (RepR->isPredicated())
- Ops.pop_back();
- return CheckScalarType(
- VPReplicateRecipe::computeScalarType(RepR->getUnderlyingInstr(), Ops));
- }
- case VPRecipeBase::VPWidenSC: {
- SmallVector<VPValue *, 4> Ops(R.operands());
- return CheckScalarType(computeScalarTypeForInstruction(
- cast<VPWidenRecipe>(&R)->getOpcode(), Ops));
- }
- case VPRecipeBase::VPExpressionSC:
- return CheckScalarType(cast<VPExpressionRecipe>(&R)
- ->getOperandOfResultType()
- ->getScalarType());
- default:
- return true;
- }
- llvm_unreachable("all recipes must be handled above");
-}
-
bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!verifyPhiRecipes(VPBB))
return false;
@@ -328,8 +231,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
errs() << "not in a VPIRBasicBlock!\n";
return false;
}
- if (!verifyRecipeTypes(R))
- return false;
for (const VPValue *V : R.definedValues()) {
// Verify that each defined value has a scalar type.
if (!V->getScalarType()) {
More information about the llvm-commits
mailing list