[llvm] [VPlan] Move recording of Inst->VPValue to VPRecipeBuilder (NFCI). (PR #84464)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 18 12:32:40 PDT 2024
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/84464
>From b09a411e5b14a566aa0a79c4d6e6624781583b4b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 8 Mar 2024 11:17:39 +0000
Subject: [PATCH 1/3] [VPlan] Move recording of Inst->VPValue to
VPRecipeBuilder (NFCI).
Instead of keeping a mapping of Inst->VPValues (of their corresponding
recipes) in VPlan's Value2VPValue mapping, keep it in VPRecipeBuilder
instead. After recently replacing the last user of this mapping after
initial construction, this mapping is only needed for recipe
construction (to map IR operands to VPValue operands).
By moving the mapping, VPlan's VPValue tracking can be simplified and
limited only to live-ins. It also allows removing disableValue2VPValue
and associated machinery & asserts.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 69 ++++++++-----------
.../Transforms/Vectorize/VPRecipeBuilder.h | 33 +++++----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 10 +--
llvm/lib/Transforms/Vectorize/VPlan.h | 33 ++-------
.../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 4 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 ++-
.../Transforms/Vectorize/VPlanHCFGTest.cpp | 2 +-
.../Transforms/Vectorize/VPlanTest.cpp | 4 +-
8 files changed, 65 insertions(+), 99 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index edaad4d033bdf0..f52843c7e2d8c3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7913,6 +7913,18 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
}
}
+iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
+VPRecipeBuilder::mapToVPValues(User::op_range Operands, VPlan &Plan) {
+ std::function<VPValue *(Value *)> Fn = [this, &Plan](Value *Op) {
+ if (auto *I = dyn_cast<Instruction>(Op)) {
+ if (auto *R = Ingredient2Recipe.lookup(I))
+ return R->getVPSingleValue();
+ }
+ return Plan.getOrAddLiveIn(Op);
+ };
+ return map_range(Operands, Fn);
+}
+
VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
VPlan &Plan) {
assert(is_contained(predecessors(Dst), Src) && "Invalid edge");
@@ -7938,7 +7950,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
if (OrigLoop->isLoopExiting(Src))
return EdgeMaskCache[Edge] = SrcMask;
- VPValue *EdgeMask = Plan.getVPValueOrAddLiveIn(BI->getCondition());
+ VPValue *EdgeMask = getVPValue(BI->getCondition(), Plan);
assert(EdgeMask && "No Edge Mask found for condition");
if (BI->getSuccessor(0) != Dst)
@@ -7949,7 +7961,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
// 'select i1 SrcMask, i1 EdgeMask, i1 false'.
// The select version does not introduce new UB if SrcMask is false and
// EdgeMask is poison. Using 'and' here introduces undefined behavior.
- VPValue *False = Plan.getVPValueOrAddLiveIn(
+ VPValue *False = Plan.getOrAddLiveIn(
ConstantInt::getFalse(BI->getCondition()->getType()));
EdgeMask =
Builder.createSelect(SrcMask, EdgeMask, False, BI->getDebugLoc());
@@ -8151,7 +8163,7 @@ VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
auto *Phi = cast<PHINode>(I->getOperand(0));
const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
- VPValue *Start = Plan.getVPValueOrAddLiveIn(II.getStartValue());
+ VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
*OrigLoop, Range);
}
@@ -8263,7 +8275,7 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
if (Legal->isMaskRequired(CI))
Mask = getBlockInMask(CI->getParent());
else
- Mask = Plan->getVPValueOrAddLiveIn(ConstantInt::getTrue(
+ Mask = Plan->getOrAddLiveIn(ConstantInt::getTrue(
IntegerType::getInt1Ty(Variant->getFunctionType()->getContext())));
Ops.insert(Ops.begin() + *MaskPos, Mask);
@@ -8306,8 +8318,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
if (CM.isPredicatedInst(I)) {
SmallVector<VPValue *> Ops(Operands.begin(), Operands.end());
VPValue *Mask = getBlockInMask(I->getParent());
- VPValue *One = Plan->getVPValueOrAddLiveIn(
- ConstantInt::get(I->getType(), 1u, false));
+ VPValue *One =
+ Plan->getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
auto *SafeRHS =
new VPInstruction(Instruction::Select, {Mask, Ops[1], One},
I->getDebugLoc());
@@ -8402,7 +8414,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
BlockInMask = getBlockInMask(I->getParent());
}
- auto *Recipe = new VPReplicateRecipe(I, Plan.mapToVPValues(I->operands()),
+ auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands(), Plan),
IsUniform, BlockInMask);
return Recipe;
}
@@ -8417,10 +8429,6 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
if (Phi->getParent() != OrigLoop->getHeader())
return tryToBlend(Phi, Operands, Plan);
- // Always record recipes for header phis. Later first-order recurrence phis
- // can have earlier phis as incoming values.
- recordRecipeOf(Phi);
-
if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, *Plan, Range)))
return Recipe;
@@ -8445,14 +8453,6 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV);
}
- // Record the incoming value from the backedge, so we can add the incoming
- // value from the backedge after all recipes have been created.
- auto *Inc = cast<Instruction>(
- Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
- auto RecipeIter = Ingredient2Recipe.find(Inc);
- if (RecipeIter == Ingredient2Recipe.end())
- recordRecipeOf(Inc);
-
PhisToFix.push_back(PhiRecipe);
return PhiRecipe;
}
@@ -8518,7 +8518,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
DebugLoc DL) {
Value *StartIdx = ConstantInt::get(IdxTy, 0);
- auto *StartV = Plan.getVPValueOrAddLiveIn(StartIdx);
+ auto *StartV = Plan.getOrAddLiveIn(StartIdx);
// Add a VPCanonicalIVPHIRecipe starting at 0 to the header.
auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DL);
@@ -8541,7 +8541,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
// Add exit values to \p Plan. VPLiveOuts are added for each LCSSA phi in the
// original exit block.
static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop,
- VPlan &Plan) {
+ VPRecipeBuilder &Builder, VPlan &Plan) {
BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock();
BasicBlock *ExitingBB = OrigLoop->getExitingBlock();
// Only handle single-exit loops with unique exit blocks for now.
@@ -8552,7 +8552,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop,
for (PHINode &ExitPhi : ExitBB->phis()) {
Value *IncomingValue =
ExitPhi.getIncomingValueForBlock(ExitingBB);
- VPValue *V = Plan.getVPValueOrAddLiveIn(IncomingValue);
+ VPValue *V = Builder.getVPValue(IncomingValue, Plan);
Plan.addLiveOut(&ExitPhi, V);
}
}
@@ -8588,9 +8588,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
if (!getDecisionAndClampRange(applyIG, Range))
continue;
InterleaveGroups.insert(IG);
- for (unsigned i = 0; i < IG->getFactor(); i++)
- if (Instruction *Member = IG->getMember(i))
- RecipeBuilder.recordRecipeOf(Member);
};
// ---------------------------------------------------------------------------
@@ -8659,10 +8656,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
SmallVector<VPValue *, 4> Operands;
auto *Phi = dyn_cast<PHINode>(Instr);
if (Phi && Phi->getParent() == HeaderBB) {
- Operands.push_back(Plan->getVPValueOrAddLiveIn(
+ Operands.push_back(Plan->getOrAddLiveIn(
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
} else {
- auto OpRange = Plan->mapToVPValues(Instr->operands());
+ auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands(), *Plan);
Operands = {OpRange.begin(), OpRange.end()};
}
@@ -8677,10 +8674,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Instr, Operands, Range, VPBB, Plan);
if (!Recipe)
Recipe = RecipeBuilder.handleReplication(Instr, Range, *Plan);
- for (auto *Def : Recipe->definedValues()) {
- auto *UV = Def->getUnderlyingValue();
- Plan->addVPValue(UV, Def);
- }
RecipeBuilder.setRecipe(Instr, Recipe);
if (isa<VPHeaderPHIRecipe>(Recipe)) {
@@ -8712,7 +8705,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
// and there is nothing to fix from vector loop; phis should have incoming
// from scalar loop only.
} else
- addUsersInExitBlock(HeaderVPBB, OrigLoop, *Plan);
+ addUsersInExitBlock(HeaderVPBB, OrigLoop, RecipeBuilder, *Plan);
assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
!Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
@@ -8774,16 +8767,12 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
continue;
Constant *CI = ConstantInt::get(Stride->getType(), ScevStride->getAPInt());
- auto *ConstVPV = Plan->getVPValueOrAddLiveIn(CI);
+ auto *ConstVPV = Plan->getOrAddLiveIn(CI);
// The versioned value may not be used in the loop directly, so just add a
// new live-in in those cases.
- Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
+ Plan->getOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
}
- // From this point onwards, VPlan-to-VPlan transformations may change the plan
- // in ways that accessing values using original IR values is incorrect.
- Plan->disableValue2VPValue();
-
VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) {
return Legal->blockNeedsPredication(BB);
});
@@ -10082,7 +10071,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogILV.setTripCount(MainILV.getTripCount());
for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
auto *ExpandR = cast<VPExpandSCEVRecipe>(&R);
- auto *ExpandedVal = BestEpiPlan.getVPValueOrAddLiveIn(
+ auto *ExpandedVal = BestEpiPlan.getOrAddLiveIn(
ExpandedSCEVs.find(ExpandR->getSCEV())->second);
ExpandR->replaceAllUsesWith(ExpandedVal);
if (BestEpiPlan.getTripCount() == ExpandR)
@@ -10123,7 +10112,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
{EPI.MainLoopIterationCountCheck});
}
assert(ResumeV && "Must have a resume value");
- VPValue *StartVal = BestEpiPlan.getVPValueOrAddLiveIn(ResumeV);
+ VPValue *StartVal = BestEpiPlan.getOrAddLiveIn(ResumeV);
cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index b1498026adadfe..527417e29906fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -49,9 +49,8 @@ class VPRecipeBuilder {
EdgeMaskCacheTy EdgeMaskCache;
BlockMaskCacheTy BlockMaskCache;
- // VPlan-VPlan transformations support: Hold a mapping from ingredients to
- // their recipe. To save on memory, only do so for selected ingredients,
- // marked by having a nullptr entry in this map.
+ // VPlan construction support: Hold a mapping from ingredients to
+ // their recipe.
DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
/// Cross-iteration reduction & first-order recurrence phis for which we need
@@ -117,13 +116,8 @@ class VPRecipeBuilder {
VFRange &Range, VPBasicBlock *VPBB,
VPlanPtr &Plan);
- /// Set the recipe created for given ingredient. This operation is a no-op for
- /// ingredients that were not marked using a nullptr entry in the map.
+ /// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
- if (!Ingredient2Recipe.count(I))
- return;
- assert(Ingredient2Recipe[I] == nullptr &&
- "Recipe already set for ingredient");
Ingredient2Recipe[I] = R;
}
@@ -146,14 +140,6 @@ class VPRecipeBuilder {
/// between SRC and DST.
VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const;
- /// Mark given ingredient for recording its recipe once one is created for
- /// it.
- void recordRecipeOf(Instruction *I) {
- assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) &&
- "Recipe already set for ingredient");
- Ingredient2Recipe[I] = nullptr;
- }
-
/// Return the recipe created for given ingredient.
VPRecipeBase *getRecipe(Instruction *I) {
assert(Ingredient2Recipe.count(I) &&
@@ -172,6 +158,19 @@ class VPRecipeBuilder {
/// Add the incoming values from the backedge to reduction & first-order
/// recurrence cross-iteration phis.
void fixHeaderPhis();
+
+ /// Returns a range mapping the values the range \p Operands to their
+ /// corresponding VPValues.
+ iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
+ mapToVPValues(User::op_range Operands, VPlan &Plan);
+
+ VPValue *getVPValue(Value *V, VPlan &Plan) {
+ if (auto *I = dyn_cast<Instruction>(V)) {
+ if (auto *R = Ingredient2Recipe.lookup(I))
+ return R->getVPSingleValue();
+ }
+ return Plan.getOrAddLiveIn(V);
+ }
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9768e4b7aa0a8a..53ee9793fdefe9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -812,7 +812,7 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
// needs to be changed from zero to the value after the main vector loop.
// FIXME: Improve modeling for canonical IV start values in the epilogue loop.
if (CanonicalIVStartValue) {
- VPValue *VPV = getVPValueOrAddLiveIn(CanonicalIVStartValue);
+ VPValue *VPV = getOrAddLiveIn(CanonicalIVStartValue);
auto *IV = getCanonicalIV();
assert(all_of(IV->users(),
[](const VPUser *U) {
@@ -1091,7 +1091,7 @@ VPlan *VPlan::duplicate() {
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
for (VPValue *OldLiveIn : VPLiveInsToFree) {
Old2NewVPValues[OldLiveIn] =
- NewPlan->getVPValueOrAddLiveIn(OldLiveIn->getLiveInIRValue());
+ NewPlan->getOrAddLiveIn(OldLiveIn->getLiveInIRValue());
}
Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount;
Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
@@ -1102,7 +1102,7 @@ VPlan *VPlan::duplicate() {
assert(TripCount && "trip count must be set");
if (TripCount->isLiveIn())
Old2NewVPValues[TripCount] =
- NewPlan->getVPValueOrAddLiveIn(TripCount->getLiveInIRValue());
+ NewPlan->getOrAddLiveIn(TripCount->getLiveInIRValue());
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
@@ -1425,9 +1425,9 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
return Expanded;
VPValue *Expanded = nullptr;
if (auto *E = dyn_cast<SCEVConstant>(Expr))
- Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
+ Expanded = Plan.getOrAddLiveIn(E->getValue());
else if (auto *E = dyn_cast<SCEVUnknown>(Expr))
- Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
+ Expanded = Plan.getOrAddLiveIn(E->getValue());
else {
Expanded = new VPExpandSCEVRecipe(Expr, SE);
Plan.getPreheader()->appendRecipe(Expanded->getDefiningRecipe());
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index af6d0081bffebc..26162c25664e5b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2870,10 +2870,6 @@ class VPlan {
/// definitions are VPValues that hold a pointer to their underlying IR.
SmallVector<VPValue *, 16> VPLiveInsToFree;
- /// Indicates whether it is safe use the Value2VPValue mapping or if the
- /// mapping cannot be used any longer, because it is stale.
- bool Value2VPValueEnabled = true;
-
/// Values used outside the plan.
MapVector<PHINode *, VPLiveOut *> LiveOuts;
@@ -2952,10 +2948,6 @@ class VPlan {
/// Returns VF * UF of the vector loop region.
VPValue &getVFxUF() { return VFxUF; }
- /// Mark the plan to indicate that using Value2VPValue is not safe any
- /// longer, because it may be stale.
- void disableValue2VPValue() { Value2VPValueEnabled = false; }
-
void addVF(ElementCount VF) { VFs.insert(VF); }
void setVF(ElementCount VF) {
@@ -2985,25 +2977,22 @@ class VPlan {
void setName(const Twine &newName) { Name = newName.str(); }
void addVPValue(Value *V, VPValue *VPV) {
- assert((Value2VPValueEnabled || VPV->isLiveIn()) &&
- "Value2VPValue mapping may be out of date!");
- assert(V && "Trying to add a null Value to VPlan");
- assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
+ assert(VPV->isLiveIn());
Value2VPValue[V] = VPV;
}
/// Returns the VPValue for \p V.
- VPValue *getVPValue(Value *V) {
+ VPValue *getLiveIn(Value *V) {
assert(V && "Trying to get the VPValue of a null Value");
assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
- assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) &&
- "Value2VPValue mapping may be out of date!");
+ assert(Value2VPValue[V]->isLiveIn() &&
+ "Only live-ins should be in mapping");
return Value2VPValue[V];
}
/// Gets the VPValue for \p V or adds a new live-in (if none exists yet) for
/// \p V.
- VPValue *getVPValueOrAddLiveIn(Value *V) {
+ VPValue *getOrAddLiveIn(Value *V) {
assert(V && "Trying to get or add the VPValue of a null Value");
if (!Value2VPValue.count(V)) {
VPValue *VPV = new VPValue(V);
@@ -3011,7 +3000,7 @@ class VPlan {
addVPValue(V, VPV);
}
- return getVPValue(V);
+ return getLiveIn(V);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -3028,16 +3017,6 @@ class VPlan {
LLVM_DUMP_METHOD void dump() const;
#endif
- /// Returns a range mapping the values the range \p Operands to their
- /// corresponding VPValues.
- iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
- mapToVPValues(User::op_range Operands) {
- std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
- return getVPValueOrAddLiveIn(Op);
- };
- return map_range(Operands, Fn);
- }
-
/// Returns the VPRegionBlock of the vector loop.
VPRegionBlock *getVectorLoopRegion() {
return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 6474a9697dce89..da246869625bb0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -272,7 +272,7 @@ VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
// A and B: Create VPValue and add it to the pool of external definitions and
// to the Value->VPValue map.
- VPValue *NewVPVal = Plan.getVPValueOrAddLiveIn(IRVal);
+ VPValue *NewVPVal = Plan.getOrAddLiveIn(IRVal);
IRDef2VPValue[IRVal] = NewVPVal;
return NewVPVal;
}
@@ -362,7 +362,7 @@ void PlainCFGBuilder::buildPlainCFG() {
for (auto &I : *ThePreheaderBB) {
if (I.getType()->isVoidTy())
continue;
- IRDef2VPValue[&I] = Plan.getVPValueOrAddLiveIn(&I);
+ IRDef2VPValue[&I] = Plan.getOrAddLiveIn(&I);
}
LoopBlocksRPO RPO(TheLoop);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f6b564ad931ca9..93cbe7065db572 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -49,12 +49,11 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
- VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue());
+ VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II);
} else {
- Plan->addVPValue(Phi, VPPhi);
continue;
}
} else {
@@ -625,9 +624,9 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
return;
LLVMContext &Ctx = SE.getContext();
- auto *BOC = new VPInstruction(
- VPInstruction::BranchOnCond,
- {Plan.getVPValueOrAddLiveIn(ConstantInt::getTrue(Ctx))});
+ auto *BOC =
+ new VPInstruction(VPInstruction::BranchOnCond,
+ {Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx))});
Term->eraseFromParent();
ExitingVPBB->appendRecipe(BOC);
Plan.setVF(BestVF);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index be8be7acbe3881..777675b623f32a 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -96,7 +96,7 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
// Add an external value to check we do not print the list of external values,
// as this is not required with the new printing.
- Plan->getVPValueOrAddLiveIn(&*F->arg_begin());
+ Plan->getOrAddLiveIn(&*F->arg_begin());
std::string FullDump;
raw_string_ostream OS(FullDump);
Plan->printDOT(OS);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 33d5e2759af590..e537aac75515ca 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1237,8 +1237,8 @@ TEST(VPRecipeTest, dump) {
BinaryOperator::CreateAdd(UndefValue::get(Int32), UndefValue::get(Int32));
AI->setName("a");
SmallVector<VPValue *, 2> Args;
- VPValue *ExtVPV1 = Plan.getVPValueOrAddLiveIn(ConstantInt::get(Int32, 1));
- VPValue *ExtVPV2 = Plan.getVPValueOrAddLiveIn(ConstantInt::get(Int32, 2));
+ VPValue *ExtVPV1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+ VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
VPWidenRecipe *WidenR =
>From c509520cb172bcd99dc511ce86d3052462361260 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 11 Mar 2024 21:43:29 +0000
Subject: [PATCH 2/3] !fixup address comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 28 +++++++++----------
.../Transforms/Vectorize/VPRecipeBuilder.h | 8 ++++--
llvm/lib/Transforms/Vectorize/VPlan.cpp | 10 +++----
llvm/lib/Transforms/Vectorize/VPlan.h | 10 ++++---
.../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 4 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 +++---
.../Transforms/Vectorize/VPlanHCFGTest.cpp | 2 +-
.../Transforms/Vectorize/VPlanTest.cpp | 4 +--
8 files changed, 39 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f52843c7e2d8c3..f7670f969acab5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7920,7 +7920,7 @@ VPRecipeBuilder::mapToVPValues(User::op_range Operands, VPlan &Plan) {
if (auto *R = Ingredient2Recipe.lookup(I))
return R->getVPSingleValue();
}
- return Plan.getOrAddLiveIn(Op);
+ return Plan.getVPValueOrAddLiveIn(Op);
};
return map_range(Operands, Fn);
}
@@ -7950,7 +7950,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
if (OrigLoop->isLoopExiting(Src))
return EdgeMaskCache[Edge] = SrcMask;
- VPValue *EdgeMask = getVPValue(BI->getCondition(), Plan);
+ VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan);
assert(EdgeMask && "No Edge Mask found for condition");
if (BI->getSuccessor(0) != Dst)
@@ -7961,7 +7961,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
// 'select i1 SrcMask, i1 EdgeMask, i1 false'.
// The select version does not introduce new UB if SrcMask is false and
// EdgeMask is poison. Using 'and' here introduces undefined behavior.
- VPValue *False = Plan.getOrAddLiveIn(
+ VPValue *False = Plan.getVPValueOrAddLiveIn(
ConstantInt::getFalse(BI->getCondition()->getType()));
EdgeMask =
Builder.createSelect(SrcMask, EdgeMask, False, BI->getDebugLoc());
@@ -8163,7 +8163,7 @@ VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
auto *Phi = cast<PHINode>(I->getOperand(0));
const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
- VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
+ VPValue *Start = Plan.getVPValueOrAddLiveIn(II.getStartValue());
return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
*OrigLoop, Range);
}
@@ -8275,7 +8275,7 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
if (Legal->isMaskRequired(CI))
Mask = getBlockInMask(CI->getParent());
else
- Mask = Plan->getOrAddLiveIn(ConstantInt::getTrue(
+ Mask = Plan->getVPValueOrAddLiveIn(ConstantInt::getTrue(
IntegerType::getInt1Ty(Variant->getFunctionType()->getContext())));
Ops.insert(Ops.begin() + *MaskPos, Mask);
@@ -8318,8 +8318,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
if (CM.isPredicatedInst(I)) {
SmallVector<VPValue *> Ops(Operands.begin(), Operands.end());
VPValue *Mask = getBlockInMask(I->getParent());
- VPValue *One =
- Plan->getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
+ VPValue *One = Plan->getVPValueOrAddLiveIn(
+ ConstantInt::get(I->getType(), 1u, false));
auto *SafeRHS =
new VPInstruction(Instruction::Select, {Mask, Ops[1], One},
I->getDebugLoc());
@@ -8518,7 +8518,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
DebugLoc DL) {
Value *StartIdx = ConstantInt::get(IdxTy, 0);
- auto *StartV = Plan.getOrAddLiveIn(StartIdx);
+ auto *StartV = Plan.getVPValueOrAddLiveIn(StartIdx);
// Add a VPCanonicalIVPHIRecipe starting at 0 to the header.
auto *CanonicalIVPHI = new VPCanonicalIVPHIRecipe(StartV, DL);
@@ -8552,7 +8552,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop,
for (PHINode &ExitPhi : ExitBB->phis()) {
Value *IncomingValue =
ExitPhi.getIncomingValueForBlock(ExitingBB);
- VPValue *V = Builder.getVPValue(IncomingValue, Plan);
+ VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan);
Plan.addLiveOut(&ExitPhi, V);
}
}
@@ -8656,7 +8656,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
SmallVector<VPValue *, 4> Operands;
auto *Phi = dyn_cast<PHINode>(Instr);
if (Phi && Phi->getParent() == HeaderBB) {
- Operands.push_back(Plan->getOrAddLiveIn(
+ Operands.push_back(Plan->getVPValueOrAddLiveIn(
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
} else {
auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands(), *Plan);
@@ -8767,10 +8767,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
continue;
Constant *CI = ConstantInt::get(Stride->getType(), ScevStride->getAPInt());
- auto *ConstVPV = Plan->getOrAddLiveIn(CI);
+ auto *ConstVPV = Plan->getVPValueOrAddLiveIn(CI);
// The versioned value may not be used in the loop directly, so just add a
// new live-in in those cases.
- Plan->getOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
+ Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
}
VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) {
@@ -10071,7 +10071,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogILV.setTripCount(MainILV.getTripCount());
for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
auto *ExpandR = cast<VPExpandSCEVRecipe>(&R);
- auto *ExpandedVal = BestEpiPlan.getOrAddLiveIn(
+ auto *ExpandedVal = BestEpiPlan.getVPValueOrAddLiveIn(
ExpandedSCEVs.find(ExpandR->getSCEV())->second);
ExpandR->replaceAllUsesWith(ExpandedVal);
if (BestEpiPlan.getTripCount() == ExpandR)
@@ -10112,7 +10112,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
{EPI.MainLoopIterationCountCheck});
}
assert(ResumeV && "Must have a resume value");
- VPValue *StartVal = BestEpiPlan.getOrAddLiveIn(ResumeV);
+ VPValue *StartVal = BestEpiPlan.getVPValueOrAddLiveIn(ResumeV);
cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 527417e29906fe..4a7520e87e32af 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -118,6 +118,8 @@ class VPRecipeBuilder {
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
+ assert(!Ingredient2Recipe.contains(I) &&
+ "Cannot reset recipe for instruction.");
Ingredient2Recipe[I] = R;
}
@@ -159,17 +161,17 @@ class VPRecipeBuilder {
/// recurrence cross-iteration phis.
void fixHeaderPhis();
- /// Returns a range mapping the values the range \p Operands to their
+ /// Returns a range mapping the values of the range \p Operands to their
/// corresponding VPValues.
iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
mapToVPValues(User::op_range Operands, VPlan &Plan);
- VPValue *getVPValue(Value *V, VPlan &Plan) {
+ VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) {
if (auto *I = dyn_cast<Instruction>(V)) {
if (auto *R = Ingredient2Recipe.lookup(I))
return R->getVPSingleValue();
}
- return Plan.getOrAddLiveIn(V);
+ return Plan.getVPValueOrAddLiveIn(V);
}
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 53ee9793fdefe9..9768e4b7aa0a8a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -812,7 +812,7 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
// needs to be changed from zero to the value after the main vector loop.
// FIXME: Improve modeling for canonical IV start values in the epilogue loop.
if (CanonicalIVStartValue) {
- VPValue *VPV = getOrAddLiveIn(CanonicalIVStartValue);
+ VPValue *VPV = getVPValueOrAddLiveIn(CanonicalIVStartValue);
auto *IV = getCanonicalIV();
assert(all_of(IV->users(),
[](const VPUser *U) {
@@ -1091,7 +1091,7 @@ VPlan *VPlan::duplicate() {
DenseMap<VPValue *, VPValue *> Old2NewVPValues;
for (VPValue *OldLiveIn : VPLiveInsToFree) {
Old2NewVPValues[OldLiveIn] =
- NewPlan->getOrAddLiveIn(OldLiveIn->getLiveInIRValue());
+ NewPlan->getVPValueOrAddLiveIn(OldLiveIn->getLiveInIRValue());
}
Old2NewVPValues[&VectorTripCount] = &NewPlan->VectorTripCount;
Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
@@ -1102,7 +1102,7 @@ VPlan *VPlan::duplicate() {
assert(TripCount && "trip count must be set");
if (TripCount->isLiveIn())
Old2NewVPValues[TripCount] =
- NewPlan->getOrAddLiveIn(TripCount->getLiveInIRValue());
+ NewPlan->getVPValueOrAddLiveIn(TripCount->getLiveInIRValue());
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
@@ -1425,9 +1425,9 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
return Expanded;
VPValue *Expanded = nullptr;
if (auto *E = dyn_cast<SCEVConstant>(Expr))
- Expanded = Plan.getOrAddLiveIn(E->getValue());
+ Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
else if (auto *E = dyn_cast<SCEVUnknown>(Expr))
- Expanded = Plan.getOrAddLiveIn(E->getValue());
+ Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
else {
Expanded = new VPExpandSCEVRecipe(Expr, SE);
Plan.getPreheader()->appendRecipe(Expanded->getDefiningRecipe());
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 26162c25664e5b..9499d808e8b274 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2977,12 +2977,14 @@ class VPlan {
void setName(const Twine &newName) { Name = newName.str(); }
void addVPValue(Value *V, VPValue *VPV) {
- assert(VPV->isLiveIn());
+ assert(VPV->isLiveIn() && "VPV must be a live-in.");
+ assert(V && "Trying to add a null Value to VPlan");
+ assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
Value2VPValue[V] = VPV;
}
/// Returns the VPValue for \p V.
- VPValue *getLiveIn(Value *V) {
+ VPValue *getVPValue(Value *V) {
assert(V && "Trying to get the VPValue of a null Value");
assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
assert(Value2VPValue[V]->isLiveIn() &&
@@ -2992,7 +2994,7 @@ class VPlan {
/// Gets the VPValue for \p V or adds a new live-in (if none exists yet) for
/// \p V.
- VPValue *getOrAddLiveIn(Value *V) {
+ VPValue *getVPValueOrAddLiveIn(Value *V) {
assert(V && "Trying to get or add the VPValue of a null Value");
if (!Value2VPValue.count(V)) {
VPValue *VPV = new VPValue(V);
@@ -3000,7 +3002,7 @@ class VPlan {
addVPValue(V, VPV);
}
- return getLiveIn(V);
+ return getVPValue(V);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index da246869625bb0..6474a9697dce89 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -272,7 +272,7 @@ VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) {
// A and B: Create VPValue and add it to the pool of external definitions and
// to the Value->VPValue map.
- VPValue *NewVPVal = Plan.getOrAddLiveIn(IRVal);
+ VPValue *NewVPVal = Plan.getVPValueOrAddLiveIn(IRVal);
IRDef2VPValue[IRVal] = NewVPVal;
return NewVPVal;
}
@@ -362,7 +362,7 @@ void PlainCFGBuilder::buildPlainCFG() {
for (auto &I : *ThePreheaderBB) {
if (I.getType()->isVoidTy())
continue;
- IRDef2VPValue[&I] = Plan.getOrAddLiveIn(&I);
+ IRDef2VPValue[&I] = Plan.getVPValueOrAddLiveIn(&I);
}
LoopBlocksRPO RPO(TheLoop);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 93cbe7065db572..ce630ba66a7845 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -49,7 +49,7 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
- VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
+ VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue());
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II);
@@ -624,9 +624,9 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
return;
LLVMContext &Ctx = SE.getContext();
- auto *BOC =
- new VPInstruction(VPInstruction::BranchOnCond,
- {Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx))});
+ auto *BOC = new VPInstruction(
+ VPInstruction::BranchOnCond,
+ {Plan.getVPValueOrAddLiveIn(ConstantInt::getTrue(Ctx))});
Term->eraseFromParent();
ExitingVPBB->appendRecipe(BOC);
Plan.setVF(BestVF);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 777675b623f32a..be8be7acbe3881 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -96,7 +96,7 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
// Add an external value to check we do not print the list of external values,
// as this is not required with the new printing.
- Plan->getOrAddLiveIn(&*F->arg_begin());
+ Plan->getVPValueOrAddLiveIn(&*F->arg_begin());
std::string FullDump;
raw_string_ostream OS(FullDump);
Plan->printDOT(OS);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index e537aac75515ca..33d5e2759af590 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1237,8 +1237,8 @@ TEST(VPRecipeTest, dump) {
BinaryOperator::CreateAdd(UndefValue::get(Int32), UndefValue::get(Int32));
AI->setName("a");
SmallVector<VPValue *, 2> Args;
- VPValue *ExtVPV1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+ VPValue *ExtVPV1 = Plan.getVPValueOrAddLiveIn(ConstantInt::get(Int32, 1));
+ VPValue *ExtVPV2 = Plan.getVPValueOrAddLiveIn(ConstantInt::get(Int32, 2));
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
VPWidenRecipe *WidenR =
>From ae8e89058c61e4985bc897b22afe8eafb426bd46 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 18 Mar 2024 19:30:43 +0000
Subject: [PATCH 3/3] !fixup after merge
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 +++------
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 2 +-
2 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e99fcdb6de6771..28748142ae9255 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7900,7 +7900,7 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
VPRecipeBuilder::mapToVPValues(User::op_range Operands) {
- std::function<VPValue *(Value *)> Fn = [this, &Plan](Value *Op) {
+ std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
if (auto *I = dyn_cast<Instruction>(Op)) {
if (auto *R = Ingredient2Recipe.lookup(I))
return R->getVPSingleValue();
@@ -8395,7 +8395,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
BlockInMask = getBlockInMask(I->getParent());
}
- auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands(), Plan),
+ auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()),
IsUniform, BlockInMask);
return Recipe;
}
@@ -8603,9 +8603,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
if (!getDecisionAndClampRange(applyIG, Range))
continue;
InterleaveGroups.insert(IG);
- for (unsigned i = 0; i < IG->getFactor(); i++)
- if (Instruction *Member = IG->getMember(i))
- RecipeBuilder.recordRecipeOf(Member);
};
// ---------------------------------------------------------------------------
@@ -8647,7 +8644,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Operands.push_back(Plan->getVPValueOrAddLiveIn(
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
} else {
- auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands(), *Plan);
+ auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands());
Operands = {OpRange.begin(), OpRange.end()};
}
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 0b0ca237763b93..fbb70f333685d7 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -163,7 +163,7 @@ class VPRecipeBuilder {
/// Returns a range mapping the values of the range \p Operands to their
/// corresponding VPValues.
iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
- mapToVPValues(User::op_range Operands, VPlan &Plan);
+ mapToVPValues(User::op_range Operands);
VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) {
if (auto *I = dyn_cast<Instruction>(V)) {
More information about the llvm-commits
mailing list