[llvm] f3a8165 - [VPlan] Add VPSymbolicValue for UF. (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 15 07:24:52 PST 2026
Author: Florian Hahn
Date: 2026-02-15T15:24:35Z
New Revision: f3a816598dc281666fbe799776656249f52afda3
URL: https://github.com/llvm/llvm-project/commit/f3a816598dc281666fbe799776656249f52afda3
DIFF: https://github.com/llvm/llvm-project/commit/f3a816598dc281666fbe799776656249f52afda3.diff
LOG: [VPlan] Add VPSymbolicValue for UF. (NFC)
Add a symbolic unroll factor (UF) to VPlan similar to VF & VFxUF that
gets replaced with the concrete UF during plan execution, similar to how VF
is used for the vectorization factor. This is a preparatory change that
allows transforms to use the symbolic UF before the concrete UF is
determined.
Note that the old getUF that returns the concrete UF after unrolling has
been renamed to getConcreteUF.
Split off from the re-commit of 8d29d093096
(https://github.com/llvm/llvm-project/pull/149706) as suggested.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b5feb3feea2e1..a1971683cfdf6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7467,7 +7467,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
VPlanTransforms::materializeVectorTripCount(
BestVPlan, VectorPH, CM.foldTailByMasking(),
CM.requiresScalarEpilogue(BestVF.isVector()));
- VPlanTransforms::materializeVFAndVFxUF(BestVPlan, VectorPH, BestVF);
+ VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
VPlanTransforms::cse(BestVPlan);
VPlanTransforms::simplifyRecipes(BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index c266842f107e8..e4b05a410d303 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -346,7 +346,7 @@ void VPTransformState::setDebugLocFrom(DebugLoc DL) {
->shouldEmitDebugInfoForProfiling() &&
!EnableFSDiscriminator) {
// FIXME: For scalable vectors, assume vscale=1.
- unsigned UF = Plan->getUF();
+ unsigned UF = Plan->getConcreteUF();
auto NewDIL =
DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue());
if (NewDIL)
@@ -936,7 +936,7 @@ void VPlan::execute(VPTransformState *State) {
{{DominatorTree::Delete, VectorPreHeader, State->CFG.ExitBB}});
LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF
- << ", UF=" << getUF() << '\n');
+ << ", UF=" << getConcreteUF() << '\n');
setName("Final VPlan");
// TODO: RUN_VPLAN_PASS/VPlanTransforms::runPass should automatically dump
// VPlans after some specific stages when "-debug" is specified, but that
@@ -1055,6 +1055,12 @@ void VPlan::printLiveIns(raw_ostream &O) const {
O << " = VF";
}
+ if (UF.getNumUsers() > 0) {
+ O << "\nLive-in ";
+ UF.printAsOperand(O, SlotTracker);
+ O << " = UF";
+ }
+
if (VFxUF.getNumUsers() > 0) {
O << "\nLive-in ";
VFxUF.printAsOperand(O, SlotTracker);
@@ -1195,6 +1201,7 @@ VPlan *VPlan::duplicate() {
Old2NewVPValues[OldLiveIn] = NewPlan->getOrAddLiveIn(OldLiveIn);
Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount;
Old2NewVPValues[&VF] = &NewPlan->VF;
+ Old2NewVPValues[&UF] = &NewPlan->UF;
Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
if (BackedgeTakenCount) {
NewPlan->BackedgeTakenCount = new VPSymbolicValue();
@@ -1484,6 +1491,8 @@ void VPSlotTracker::assignName(const VPValue *V) {
void VPSlotTracker::assignNames(const VPlan &Plan) {
if (Plan.VF.getNumUsers() > 0)
assignName(&Plan.VF);
+ if (Plan.UF.getNumUsers() > 0)
+ assignName(&Plan.UF);
if (Plan.VFxUF.getNumUsers() > 0)
assignName(&Plan.VFxUF);
assignName(&Plan.VectorTripCount);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 07b6c306fa980..7f958461f0ec9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4519,6 +4519,9 @@ class VPlan {
/// Represents the vectorization factor of the loop.
VPSymbolicValue VF;
+ /// Represents the unroll factor of the loop.
+ VPSymbolicValue UF;
+
/// Represents the loop-invariant VF * UF of the vector loop region.
VPSymbolicValue VFxUF;
@@ -4658,6 +4661,9 @@ class VPlan {
VPValue &getVF() { return VF; };
const VPValue &getVF() const { return VF; };
+ /// Returns the UF of the vector loop region.
+ VPValue &getUF() { return UF; };
+
/// Returns VF * UF of the vector loop region.
VPValue &getVFxUF() { return VFxUF; }
@@ -4693,7 +4699,8 @@ class VPlan {
bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
- unsigned getUF() const {
+ /// Returns the concrete UF of the plan, after unrolling.
+ unsigned getConcreteUF() const {
assert(UFs.size() == 1 && "Expected a single UF");
return UFs[0];
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c15b9fb23060a..bab1fc773fa4b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -627,7 +627,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateVectorSpliceRight(V1, V2, 1, Name);
}
case VPInstruction::CalculateTripCountMinusVF: {
- unsigned UF = getParent()->getPlan()->getUF();
+ unsigned UF = getParent()->getPlan()->getConcreteUF();
Value *ScalarTC = State.get(getOperand(0), VPLane(0));
Value *Step = createStepForVF(Builder, ScalarTC->getType(), State.VF, UF);
Value *Sub = Builder.CreateSub(ScalarTC, Step);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fe2314f32d483..174e428d05c62 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1607,7 +1607,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
return Def->replaceAllUsesWith(A);
}
- if (Plan->getUF() == 1 && match(Def, m_ExtractLastPart(m_VPValue(A))))
+ if (Plan->getConcreteUF() == 1 && match(Def, m_ExtractLastPart(m_VPValue(A))))
return Def->replaceAllUsesWith(A);
}
@@ -2205,7 +2205,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
if (MadeChange) {
Plan.setVF(BestVF);
- assert(Plan.getUF() == BestUF && "BestUF must match the Plan's UF");
+ assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
}
}
@@ -5057,8 +5057,8 @@ void VPlanTransforms::materializeVectorTripCount(VPlan &Plan,
VectorTC.replaceAllUsesWith(Res);
}
-void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,
- ElementCount VFEC) {
+void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
+ ElementCount VFEC) {
VPBuilder Builder(VectorPH, VectorPH->begin());
Type *TCTy = VPTypeAnalysis(Plan).inferScalarType(Plan.getTripCount());
VPValue &VF = Plan.getVF();
@@ -5067,11 +5067,15 @@ void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,
// used.
// TODO: Assert that they aren't used.
+ VPValue *UF =
+ Plan.getOrAddLiveIn(ConstantInt::get(TCTy, Plan.getConcreteUF()));
+ Plan.getUF().replaceAllUsesWith(UF);
+
// If there are no users of the runtime VF, compute VFxUF by constant folding
// the multiplication of VF and UF.
if (VF.getNumUsers() == 0) {
VPValue *RuntimeVFxUF =
- Builder.createElementCount(TCTy, VFEC * Plan.getUF());
+ Builder.createElementCount(TCTy, VFEC * Plan.getConcreteUF());
VFxUF.replaceAllUsesWith(RuntimeVFxUF);
return;
}
@@ -5086,7 +5090,6 @@ void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,
}
VF.replaceAllUsesWith(RuntimeVF);
- VPValue *UF = Plan.getConstantInt(TCTy, Plan.getUF());
VPValue *MulByUF = Builder.createOverflowingOp(
Instruction::Mul, {RuntimeVF, UF}, {true, false});
VFxUF.replaceAllUsesWith(MulByUF);
@@ -5294,7 +5297,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
// VPTransformState.
// TODO: Remove restriction once the VF for the VectorPointer offset is
// modeled explicitly as operand.
- if (isa<VPVectorPointerRecipe>(&R) && Plan.getUF() > 1)
+ if (isa<VPVectorPointerRecipe>(&R) && Plan.getConcreteUF() > 1)
return;
// All other ops are allowed, but we reject uses that cannot be converted
@@ -5379,8 +5382,8 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
auto *Inc = cast<VPInstruction>(CanIV->getBackedgeValue());
VPBuilder PHBuilder(Plan.getVectorPreheader());
- VPValue *UF =
- Plan.getConstantInt(VectorLoop->getCanonicalIVType(), Plan.getUF());
+ VPValue *UF = Plan.getConstantInt(VectorLoop->getCanonicalIVType(),
+ Plan.getConcreteUF());
if (VF.isScalable()) {
VPValue *VScale = PHBuilder.createElementCount(
VectorLoop->getCanonicalIVType(), ElementCount::getScalable(1));
@@ -5410,7 +5413,7 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(
assert(MiddleTerm->getOpcode() == VPInstruction::BranchOnCond &&
"must have a BranchOnCond");
// Assume that `TripCount % VectorStep ` is equally distributed.
- unsigned VectorStep = Plan.getUF() * VF.getKnownMinValue();
+ unsigned VectorStep = Plan.getConcreteUF() * VF.getKnownMinValue();
if (VF.isScalable() && VScaleForTuning.has_value())
VectorStep *= *VScaleForTuning;
assert(VectorStep > 0 && "trip count should not be zero");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 5fc68c2df145f..35597c95f26f2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -413,9 +413,10 @@ struct VPlanTransforms {
/// needed.
static void materializePacksAndUnpacks(VPlan &Plan);
- /// Materialize VF and VFxUF to be computed explicitly using VPInstructions.
- static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,
- ElementCount VF);
+ /// Materialize UF, VF and VFxUF to be computed explicitly using
+ /// VPInstructions.
+ static void materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
+ ElementCount VF);
/// Expand VPExpandSCEVRecipes in \p Plan's entry block. Each
/// VPExpandSCEVRecipe is replaced with a live-in wrapping the expanded IR
More information about the llvm-commits
mailing list