[llvm] [VPlan] Thread types through VPHeaderPHIRecipe and VPDerivedIVRecipe (NFC) (PR #195894)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat May 23 10:30:51 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/195894
>From 578a4e9a143c375c4f25d1d526c0669eaa7ff952 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 19 Apr 2026 10:43:01 +0100
Subject: [PATCH 1/2] [VPlan] Thread types through VPHeaderPHIRecipe and
VPDerivedIVRecipe (NFC)
Update VPHeaderPHIRecipe and VPDerivedIVRecipe to set the scalar types
for their defined values.
This requires updating addReductionResultComputation to construct the
new chain for AnyOf reductions up-front.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 58 +++++++++++++------
llvm/lib/Transforms/Vectorize/VPlan.h | 35 ++++++-----
.../Transforms/Vectorize/VPlanAnalysis.cpp | 13 +----
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +-
.../Transforms/Vectorize/VPlanVerifier.cpp | 11 ++++
5 files changed, 74 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 363ed16f78302..ca9a285893a1f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6982,7 +6982,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPTypeAnalysis TypeInfo(*Plan);
VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
- SmallVector<VPRecipeBase *> ToDelete;
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
@@ -7011,7 +7010,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
}
auto *OrigExitingVPV = PhiR->getBackedgeValue();
- auto *NewExitingVPV = PhiR->getBackedgeValue();
+ auto *NewExitingVPV = OrigExitingVPV;
// Remove the predicated select if the target doesn't want it.
VPValue *V;
@@ -7066,24 +7065,47 @@ void LoopVectorizationPlanner::addReductionResultComputation(
if (TrueValIsPhi)
Cmp = Builder.createNot(Cmp);
- // Convert the reduction phi to operate on bools.
+ // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
+ // the exiting value flows through).
auto *NewPhiR =
- PhiR->cloneWithOperands(Plan->getFalse(), PhiR->getBackedgeValue());
+ PhiR->cloneWithOperands(Plan->getFalse(), Plan->getFalse());
NewPhiR->insertBefore(PhiR);
- PhiR->replaceAllUsesWith(NewPhiR);
-
- VPValue *Or = Builder.createOr(NewPhiR, Cmp);
- // Only replace uses inside the vector region with Or. External uses
- // (e.g. scalar preheader resume phis) must be replaced by the user
- // update loop below with FinalReductionResult.
- AnyOfSelect->replaceUsesWithIf(Or, [](VPUser &U, unsigned) {
- return cast<VPRecipeBase>(&U)->getRegion();
- });
- ToDelete.push_back(AnyOfSelect);
+ VPValue *NewExiting = Builder.createOr(NewPhiR, Cmp);
+
+ // The exiting value may flow through a VPBlendRecipe and/or a wrapping
+ // VPInstruction::Select before reaching OrigExitingVPV. Clone each level
+ // of the chain with the i1 substitutions propagated through.
+ DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
+ {PhiR, NewPhiR}};
+ auto CloneWithSubstitutions = [&](VPSingleDefRecipe *Old) {
+ SmallVector<VPValue *> NewOps;
+ for (VPValue *Op : Old->operands())
+ NewOps.push_back(Substitutions.lookup_or(Op, Op));
+ VPSingleDefRecipe *New;
+ if (auto *B = dyn_cast<VPBlendRecipe>(Old))
+ New = B->cloneWithOperands(NewOps);
+ else
+ New = cast<VPInstruction>(Old)->cloneWithOperands(NewOps);
+ New->insertBefore(Old);
+ NewExiting = New->getVPSingleValue();
+ Substitutions[Old->getVPSingleValue()] = NewExiting;
+ };
+
+ // If there's an outer Select wrapping a Blend, clone the inner Blend
+ // first so the outer Select clone can refer to it.
+ if (OrigExitingVPV != AnyOfSelect) {
+ VPValue *Inner;
+ if (match(OrigExitingVPV,
+ m_Select(m_VPValue(), m_VPValue(Inner), m_VPValue())))
+ if (auto *InnerBlend = dyn_cast<VPBlendRecipe>(Inner))
+ CloneWithSubstitutions(InnerBlend);
+ CloneWithSubstitutions(cast<VPSingleDefRecipe>(OrigExitingVPV));
+ }
- // Update NewExitingVPV if it was pointing to the now-replaced select.
- if (NewExitingVPV == AnyOfSelect)
- NewExitingVPV = Or;
+ NewPhiR->setOperand(1, NewExiting);
+ PhiR->replaceAllUsesWith(Start);
+ PhiR = NewPhiR;
+ NewExitingVPV = NewExiting;
Builder.setInsertPoint(MiddleVPBB, IP);
@@ -7166,8 +7188,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
PhiR->setOperand(0, StartV);
}
}
- for (VPRecipeBase *R : ToDelete)
- R->eraseFromParent();
RUN_VPLAN_PASS(VPlanTransforms::clearReductionWrapFlags, *Plan);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 61f9c9449c4f9..7025f39decaf5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2396,7 +2396,12 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
protected:
VPHeaderPHIRecipe(unsigned char VPRecipeID, Instruction *UnderlyingInstr,
VPValue *Start, DebugLoc DL = DebugLoc::getUnknown())
- : VPSingleDefRecipe(VPRecipeID, Start, UnderlyingInstr, DL) {}
+ : VPHeaderPHIRecipe(VPRecipeID, UnderlyingInstr, Start,
+ getScalarTypeOrInfer(Start), DL) {}
+
+ VPHeaderPHIRecipe(unsigned char VPRecipeID, Instruction *UnderlyingInstr,
+ VPValue *Start, Type *ResultTy, DebugLoc DL)
+ : VPSingleDefRecipe(VPRecipeID, Start, ResultTy, UnderlyingInstr, DL) {}
const VPRecipeBase *getAsRecipe() const override { return this; }
@@ -2465,7 +2470,13 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start,
VPValue *Step, const InductionDescriptor &IndDesc,
DebugLoc DL)
- : VPHeaderPHIRecipe(Kind, IV, Start, DL), IndDesc(IndDesc) {
+ : VPWidenInductionRecipe(Kind, IV, Start, Step, IndDesc,
+ getScalarTypeOrInfer(Start), DL) {}
+
+ VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start,
+ VPValue *Step, const InductionDescriptor &IndDesc,
+ Type *ResultTy, DebugLoc DL)
+ : VPHeaderPHIRecipe(Kind, IV, Start, ResultTy, DL), IndDesc(IndDesc) {
addOperand(Step);
}
@@ -2561,11 +2572,11 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
TruncInst *Trunc, const VPIRFlags &Flags,
DebugLoc DL)
: VPWidenInductionRecipe(VPRecipeBase::VPWidenIntOrFpInductionSC, IV,
- Start, Step, IndDesc, DL),
+ Start, Step, IndDesc,
+ Trunc ? Trunc->getType() : Start->getType(), DL),
VPIRFlags(Flags), Trunc(Trunc) {
addOperand(VF);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- (void)Metadata;
if (Trunc)
getMetadataToPropagate(Trunc, Metadata);
assert(Metadata.empty() && "unexpected metadata on Trunc");
@@ -2610,11 +2621,6 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
/// same type as the canonical induction.
bool isCanonical() const;
- /// Returns the scalar type of the induction.
- Type *getScalarType() const {
- return Trunc ? Trunc->getType() : getStartValue()->getType();
- }
-
/// Returns the VPValue representing the value of this induction at
/// the last unrolled part, if it exists. Returns itself if unrolling did not
/// take place.
@@ -2891,9 +2897,11 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
setUnderlyingValue(Phi);
}
- VPBlendRecipe *clone() override {
+ VPBlendRecipe *clone() override { return cloneWithOperands(operands()); }
+
+ VPBlendRecipe *cloneWithOperands(ArrayRef<VPValue *> NewOperands) {
return new VPBlendRecipe(cast_or_null<PHINode>(getUnderlyingValue()),
- operands(), *this, getDebugLoc());
+ NewOperands, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPBlendSC)
@@ -4045,7 +4053,8 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
const FPMathOperator *FPBinOp, VPIRValue *Start,
VPValue *IV, VPValue *Step)
- : VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, {Start, IV, Step}),
+ : VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, {Start, IV, Step},
+ Start->getScalarType(), nullptr),
Kind(Kind), FPBinOp(FPBinOp) {}
~VPDerivedIVRecipe() override = default;
@@ -4068,8 +4077,6 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
return 0;
}
- Type *getScalarType() const { return getStartValue()->getType(); }
-
VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
VPValue *getIndex() const { return getOperand(1); }
VPValue *getStepValue() const { return getOperand(2); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 96c96a2568a22..40cce07557ab5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -260,7 +260,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
VPScalarIVStepsRecipe, VPWidenCanonicalIVRecipe, VPWidenCastRecipe,
VPWidenIntrinsicRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
VPVectorEndPointerRecipe, VPWidenCallRecipe, VPWidenLoadRecipe,
- VPWidenLoadEVLRecipe>(V)) {
+ VPWidenLoadEVLRecipe, VPDerivedIVRecipe, VPHeaderPHIRecipe>(V)) {
Type *Ty = V->getScalarType();
assert(Ty && "Scalar type must be set by recipe construction");
return Ty;
@@ -268,17 +268,6 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
Type *ResultTy =
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
- .Case<VPActiveLaneMaskPHIRecipe, VPFirstOrderRecurrencePHIRecipe,
- VPReductionPHIRecipe, VPWidenPointerInductionRecipe,
- VPCurrentIterationPHIRecipe>([this](const auto *R) {
- // Handle header phi recipes, except VPWidenIntOrFpInduction
- // which needs special handling due it being possibly truncated.
- // TODO: consider inferring/caching type of siblings, e.g.,
- // backedge value, here and in cases below.
- return inferScalarType(R->getStartValue());
- })
- .Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
- [](const auto *R) { return R->getScalarType(); })
// VPInstructionWithType must be handled before VPInstruction.
.Case<VPInstructionWithType>(
[](const auto *R) { return R->getResultType(); })
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 08913c30bdbfb..e5b7041a002a1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3746,7 +3746,7 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
// The value from the original loop to which we are mapping the new induction
// variable.
- Type *Ty = TypeInfo.inferScalarType(WidenIVR);
+ Type *Ty = WidenIVR->getScalarType();
const InductionDescriptor &ID = WidenIVR->getInductionDescriptor();
Instruction::BinaryOps AddOp;
@@ -3762,7 +3762,7 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
// If the phi is truncated, truncate the start and step values.
VPBuilder Builder(Plan->getVectorPreheader());
- Type *StepTy = TypeInfo.inferScalarType(Step);
+ Type *StepTy = Step->getScalarType();
if (Ty->getScalarSizeInBits() < StepTy->getScalarSizeInBits()) {
assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
Step = Builder.createScalarCast(Instruction::Trunc, Step, Ty, DL);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 9396ea95e1922..5071aad2d0287 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -237,14 +237,25 @@ bool VPlanVerifier::verifyRecipeTypes(const VPRecipeBase &R) const {
return false;
};
+ if (auto *WII = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
+ return CheckScalarType(WII->getTruncInst()
+ ? WII->getTruncInst()->getType()
+ : WII->getStartValue()->getScalarType());
+
switch (R.getVPRecipeID()) {
case VPRecipeBase::VPVectorPointerSC:
case VPRecipeBase::VPVectorEndPointerSC:
case VPRecipeBase::VPWidenGEPSC:
case VPRecipeBase::VPScalarIVStepsSC:
+ case VPRecipeBase::VPWidenPointerInductionSC:
+ case VPRecipeBase::VPDerivedIVSC:
return CheckScalarType(getScalarTypeOrInfer(R.getOperand(0)));
case VPRecipeBase::VPWidenPHISC:
case VPRecipeBase::VPPredInstPHISC:
+ case VPRecipeBase::VPReductionPHISC:
+ case VPRecipeBase::VPActiveLaneMaskPHISC:
+ case VPRecipeBase::VPCurrentIterationPHISC:
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
return CheckOperandTypes() &&
CheckScalarType(getScalarTypeOrInfer(R.getOperand(0)));
default:
>From bacb151c4b875f9455e80cc8bfc24a4d0da02c98 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 22 May 2026 14:45:25 +0100
Subject: [PATCH 2/2] !fixup simplify a bit
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ca9a285893a1f..86694f5be04f7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7087,8 +7087,8 @@ void LoopVectorizationPlanner::addReductionResultComputation(
else
New = cast<VPInstruction>(Old)->cloneWithOperands(NewOps);
New->insertBefore(Old);
- NewExiting = New->getVPSingleValue();
- Substitutions[Old->getVPSingleValue()] = NewExiting;
+ NewExiting = New;
+ Substitutions[Old] = New;
};
// If there's an outer Select wrapping a Blend, clone the inner Blend
@@ -7101,16 +7101,13 @@ void LoopVectorizationPlanner::addReductionResultComputation(
CloneWithSubstitutions(InnerBlend);
CloneWithSubstitutions(cast<VPSingleDefRecipe>(OrigExitingVPV));
}
-
NewPhiR->setOperand(1, NewExiting);
- PhiR->replaceAllUsesWith(Start);
- PhiR = NewPhiR;
- NewExitingVPV = NewExiting;
+ PhiR->replaceAllUsesWith(
+ Plan->getOrAddLiveIn(PoisonValue::get(PhiR->getScalarType())));
Builder.setInsertPoint(MiddleVPBB, IP);
-
FinalReductionResult =
- Builder.createAnyOfReduction(NewExitingVPV, NewVal, Start, ExitDL);
+ Builder.createAnyOfReduction(NewExiting, NewVal, Start, ExitDL);
} else {
VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
PhiR->getFastMathFlags());
More information about the llvm-commits
mailing list