[llvm] fa406ba - [VPlan] Use make_isa_range instead of dyn_cast/continue (NFC) (#222770)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 10:47:11 PDT 2026
Author: Florian Hahn
Date: 2026-09-17T17:47:05Z
New Revision: fa406bab51ae60520238e555eaee51dc71fb5a76
URL: https://github.com/llvm/llvm-project/commit/fa406bab51ae60520238e555eaee51dc71fb5a76
DIFF: https://github.com/llvm/llvm-project/commit/fa406bab51ae60520238e555eaee51dc71fb5a76.diff
LOG: [VPlan] Use make_isa_range instead of dyn_cast/continue (NFC) (#222770)
Simplify loops using dyn_cast + early continue using make_isa_range,
added in https://github.com/llvm/llvm-project/pull/223414.
PR: https://github.com/llvm/llvm-project/pull/222770
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0010be774a71b..4cc704be56baa 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3349,12 +3349,11 @@ static bool hasReplicatorRegion(VPlan &Plan) {
/// Returns true if the VPlan contains a VPReductionPHIRecipe with
/// FindLast recurrence kind.
static bool hasFindLastReductionPhi(VPlan &Plan) {
- return any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- [](VPRecipeBase &R) {
- auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(&R);
- return RedPhi &&
- RecurrenceDescriptor::isFindLastRecurrenceKind(
- RedPhi->getRecurrenceKind());
+ return any_of(make_isa_range<VPReductionPHIRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
+ [](VPReductionPHIRecipe &RedPhi) {
+ return RecurrenceDescriptor::isFindLastRecurrenceKind(
+ RedPhi.getRecurrenceKind());
});
}
@@ -3745,11 +3744,9 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// Clamp the interleave ranges to reasonable counts.
bool HasUnorderedReductions =
HasReductions &&
- !any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- [](VPRecipeBase &R) {
- auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
- return RedR && RedR->isOrdered();
- });
+ !any_of(make_isa_range<VPReductionPHIRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
+ [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
unsigned MaxInterleaveCount =
TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
@@ -3913,13 +3910,13 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// do the final reduction after the loop.
bool HasSelectCmpReductions =
HasReductions &&
- any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- [](VPRecipeBase &R) {
- auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
- return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
- RedR->getRecurrenceKind()) ||
- RecurrenceDescriptor::isFindIVRecurrenceKind(
- RedR->getRecurrenceKind()));
+ any_of(make_isa_range<VPReductionPHIRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
+ [](VPReductionPHIRecipe &RedR) {
+ return RecurrenceDescriptor::isAnyOfRecurrenceKind(
+ RedR.getRecurrenceKind()) ||
+ RecurrenceDescriptor::isFindIVRecurrenceKind(
+ RedR.getRecurrenceKind());
});
if (HasSelectCmpReductions) {
LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
@@ -3933,12 +3930,9 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
// interleaving entirely.
if (HasReductions && OrigLoop->getLoopDepth() > 1) {
bool HasOrderedReductions =
- any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- [](VPRecipeBase &R) {
- auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
-
- return RedR && RedR->isOrdered();
- });
+ any_of(make_isa_range<VPReductionPHIRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
+ [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
if (HasOrderedReductions) {
LLVM_DEBUG(
dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
@@ -7249,17 +7243,15 @@ preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
bool UpdateResumePhis) {
VPBuilder Builder(Plan.getEntry());
- for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (!VPI)
- continue;
+ for (VPInstruction &VPI :
+ make_isa_range<VPInstruction>(*Plan.getMiddleBlock())) {
VPValue *OrigStart;
- if (!matchFindIVResult(VPI, m_VPValue(), m_VPValue(OrigStart)))
+ if (!matchFindIVResult(&VPI, m_VPValue(), m_VPValue(OrigStart)))
continue;
if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
continue;
VPInstruction *Freeze = Builder.createFreeze(OrigStart, {}, "fr");
- VPI->setOperand(2, Freeze);
+ VPI.setOperand(2, Freeze);
if (UpdateResumePhis)
OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
return Freeze != &U && isa<VPPhi>(&U);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 05cb353616ee5..8a7e1de318c0a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1939,17 +1939,15 @@ bool VPCostContext::useEmulatedMaskMemRefHack(const VPReplicateRecipe *R,
for (const VPBasicBlock *VPBB :
VPBlockUtils::blocksOnly<const VPBasicBlock>(
vp_depth_first_shallow(VPRB->getEntry()))) {
- for (const VPRecipeBase &Recipe : *VPBB) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
- if (!RepR)
- continue;
- if (!isa<StoreInst>(RepR->getUnderlyingInstr()))
+ for (const VPReplicateRecipe &RepR :
+ make_isa_range<VPReplicateRecipe>(*VPBB)) {
+ if (!isa<StoreInst>(RepR.getUnderlyingInstr()))
continue;
// Check if scatter is legal for this store. If so, don't count it.
- Type *Ty = RepR->getOperand(0)->getScalarType();
+ Type *Ty = RepR.getOperand(0)->getScalarType();
auto *VTy = VectorType::get(Ty, VF);
const Align Alignment =
- getLoadStoreAlignment(RepR->getUnderlyingInstr());
+ getLoadStoreAlignment(RepR.getUnderlyingInstr());
if (!TTI.isLegalMaskedScatter(VTy, Alignment))
++(*NumPredStores);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 2abfaec06dbef..af35704ed5642 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -26,12 +26,11 @@ void llvm::collectEphemeralRecipesForVPlan(
SmallVector<VPRecipeBase *> Worklist;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- if (!RepR || !match(RepR, m_Intrinsic<Intrinsic::assume>()))
+ for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(*VPBB)) {
+ if (!match(&RepR, m_Intrinsic<Intrinsic::assume>()))
continue;
- Worklist.push_back(RepR);
- EphRecipes.insert(RepR);
+ Worklist.push_back(&RepR);
+ EphRecipes.insert(&RepR);
}
}
@@ -163,11 +162,10 @@ llvm::calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef<ElementCount> VFs,
if (VPBB == LoopRegion->getExiting()) {
// VPWidenIntOrFpInductionRecipes are used implicitly at the end of the
// exiting block, where their increment will get materialized eventually.
- for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
- if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
- EndPoint[WideIV] = Idx2Recipe.size();
- Ends.insert(WideIV);
- }
+ for (auto &WideIV : make_isa_range<VPWidenIntOrFpInductionRecipe>(
+ LoopRegion->getEntryBasicBlock()->phis())) {
+ EndPoint[&WideIV] = Idx2Recipe.size();
+ Ends.insert(&WideIV);
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4cd455adb7a90..daf77a0f522d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1017,12 +1017,12 @@ bool VPlanTransforms::finalizeSCEVPredicates(VPlan &Plan,
// Collect which wide IVs have predicates and add them to PSE.
auto [HeaderVPBB, _] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
SmallPtrSet<VPWidenInductionRecipe *, 4> PredicatedIVs;
- for (auto &R : HeaderVPBB->phis()) {
- auto *WideIV = dyn_cast<VPWidenInductionRecipe>(&R);
- if (!WideIV || WideIV->getNoWrapPredicates().empty())
+ for (VPWidenInductionRecipe &WideIV :
+ make_isa_range<VPWidenInductionRecipe>(HeaderVPBB->phis())) {
+ if (WideIV.getNoWrapPredicates().empty())
continue;
- PredicatedIVs.insert(WideIV);
- for (const auto *P : WideIV->getNoWrapPredicates())
+ PredicatedIVs.insert(&WideIV);
+ for (const auto *P : WideIV.getNoWrapPredicates())
PSE.addPredicate(*P);
}
@@ -1200,9 +1200,9 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
CurrentLink->replaceAllUsesWith(RedRecipe);
// Move any store recipes using the RedRecipe that appear before it in the
// same block to just after the RedRecipe.
- for (VPUser *U : make_early_inc_range(RedRecipe->users())) {
- auto *UserR = dyn_cast<VPRecipeBase>(U);
- if (!UserR || UserR->getParent() != LinkVPBB)
+ for (VPRecipeBase *UserR : make_early_inc_range(
+ make_isa_range<VPRecipeBase>(RedRecipe->users()))) {
+ if (UserR->getParent() != LinkVPBB)
continue;
if (!match(UserR, m_VPInstruction<Instruction::Store>()))
continue;
@@ -1793,12 +1793,11 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
// result = extract-last-active vp<new.data>, vp<new.mask>, ir<default.val>
SmallVector<VPReductionPHIRecipe *, 4> Phis;
- for (VPRecipeBase &Phi :
- Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
- auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&Phi);
- if (PhiR && RecurrenceDescriptor::isFindLastRecurrenceKind(
- PhiR->getRecurrenceKind()))
- Phis.push_back(PhiR);
+ for (VPReductionPHIRecipe &PhiR : make_isa_range<VPReductionPHIRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
+ if (RecurrenceDescriptor::isFindLastRecurrenceKind(
+ PhiR.getRecurrenceKind()))
+ Phis.push_back(&PhiR);
}
if (Phis.empty())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index c9d4b3425f72f..b6ede1c1edc7f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -568,12 +568,12 @@ void VPlanTransforms::convertToVariableLengthStep(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
vp_depth_first_shallow(Plan.getEntry())))
- for (VPRecipeBase &R : VPBB->phis())
- if (auto *PhiR = dyn_cast<VPCurrentIterationPHIRecipe>(&R)) {
- assert(!CurrentIteration &&
- "Found multiple CurrentIteration. Only one expected");
- CurrentIteration = PhiR;
- }
+ for (VPCurrentIterationPHIRecipe &PhiR :
+ make_isa_range<VPCurrentIterationPHIRecipe>(VPBB->phis())) {
+ assert(!CurrentIteration &&
+ "Found multiple CurrentIteration. Only one expected");
+ CurrentIteration = &PhiR;
+ }
// Early return if it is not variable-length stepping.
if (!CurrentIteration)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index 023b3add7a837..230fb5f39d33f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -1089,18 +1089,18 @@ void VPlanTransforms::expandSCEVsToVPInstructions(VPlan &Plan,
VPSCEVExpander Expander(Builder, SE, DL);
// Expand VPExpandSCEVRecipes to VPInstructions using VPSCEVExpander.
- for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpSCEV || ExpSCEV->user_empty())
+ for (VPExpandSCEVRecipe &ExpSCEV :
+ make_early_inc_range(make_isa_range<VPExpandSCEVRecipe>(*Entry))) {
+ if (ExpSCEV.user_empty())
continue;
- Builder.setInsertPoint(ExpSCEV);
- VPValue *Expanded = Expander.expand(ExpSCEV->getSCEV());
- ExpSCEV->replaceAllUsesWith(Expanded);
+ Builder.setInsertPoint(&ExpSCEV);
+ VPValue *Expanded = Expander.expand(ExpSCEV.getSCEV());
+ ExpSCEV.replaceAllUsesWith(Expanded);
// TripCount should not be used after expansion to VPInstructions. Reset to
// poison to avoid dangling references.
- if (Plan.getTripCount() == ExpSCEV)
- Plan.resetTripCount(Plan.getPoison(ExpSCEV->getScalarType()));
- ExpSCEV->eraseFromParent();
+ if (Plan.getTripCount() == &ExpSCEV)
+ Plan.resetTripCount(Plan.getPoison(ExpSCEV.getScalarType()));
+ ExpSCEV.eraseFromParent();
}
}
@@ -1112,19 +1112,17 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
BasicBlock *EntryBB = Entry->getIRBasicBlock();
DenseMap<const SCEV *, Value *> ExpandedSCEVs;
// Expand remaining VPExpandSCEVRecipes to IR instructions using SCEVExpander.
- for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpSCEV)
- continue;
- const SCEV *Expr = ExpSCEV->getSCEV();
+ for (VPExpandSCEVRecipe &ExpSCEV :
+ make_early_inc_range(make_isa_range<VPExpandSCEVRecipe>(*Entry))) {
+ const SCEV *Expr = ExpSCEV.getSCEV();
Value *Res =
Expander.expandCodeFor(Expr, Expr->getType(), EntryBB->getTerminator());
ExpandedSCEVs[Expr] = Res;
VPValue *Exp = Plan.getOrAddLiveIn(Res);
- ExpSCEV->replaceAllUsesWith(Exp);
- if (Plan.getTripCount() == ExpSCEV)
+ ExpSCEV.replaceAllUsesWith(Exp);
+ if (Plan.getTripCount() == &ExpSCEV)
Plan.resetTripCount(Exp);
- ExpSCEV->eraseFromParent();
+ ExpSCEV.eraseFromParent();
}
assert(none_of(*Entry, IsaPred<VPExpandSCEVRecipe>) &&
"all VPExpandSCEVRecipes must have been expanded");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 85ef7ad4459d7..881c615b7a286 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -425,13 +425,10 @@ void VPPredicator::run() {
// Mask all VPInstructions in the block and record the frequency with
// which the masked recipes execute.
std::optional<VPExecutionFrequency> Freq = Frequencies.lookup(VPBB);
- for (VPRecipeBase &R : *VPBB) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (!VPI)
- continue;
- VPI->addMask(BlockMask);
- if (VPI->isMasked())
- VPI->setExecutionFrequency(Freq, Plan.getContext());
+ for (VPInstruction &VPI : make_isa_range<VPInstruction>(*VPBB)) {
+ VPI.addMask(BlockMask);
+ if (VPI.isMasked())
+ VPI.setExecutionFrequency(Freq, Plan.getContext());
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d78a3d3b1e344..eb587173f700a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -309,17 +309,16 @@ collectGroupedReplicateMemOps(
RecipesByAddressAndType;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- if (!RepR || RepR->getOpcode() != Opcode || !FilterFn(RepR))
+ for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(*VPBB)) {
+ if (RepR.getOpcode() != Opcode || !FilterFn(&RepR))
continue;
// For loads, operand 0 is address; for stores, operand 1 is address.
- VPValue *Addr = RepR->getOperand(IsLoad ? 0 : 1);
- const Type *LoadStoreTy = getLoadStoreValueType(RepR, IsLoad);
+ VPValue *Addr = RepR.getOperand(IsLoad ? 0 : 1);
+ const Type *LoadStoreTy = getLoadStoreValueType(&RepR, IsLoad);
const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(Addr, PSE, L);
if (!isa<SCEVCouldNotCompute>(AddrSCEV))
- RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(RepR);
+ RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(&RepR);
}
}
auto Groups = to_vector(RecipesByAddressAndType.values());
@@ -608,11 +607,9 @@ static void addReplicateRegions(VPlan &Plan) {
SmallVector<VPReplicateRecipe *> WorkList;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
- for (VPRecipeBase &R : *VPBB)
- if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
- if (RepR->isPredicated())
- WorkList.push_back(RepR);
- }
+ for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(*VPBB))
+ if (RepR.isPredicated())
+ WorkList.push_back(&RepR);
}
unsigned BBNum = 0;
@@ -684,9 +681,10 @@ void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
/// in the vectorized loop. There is no need to vectorize the cast - the same
/// value can be used for both the phi and casts in the vector loop.
static void removeRedundantInductionCasts(VPlan &Plan) {
- for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
- auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
- if (!IV || IV->getTruncInst())
+ for (VPWidenIntOrFpInductionRecipe &IV :
+ make_isa_range<VPWidenIntOrFpInductionRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
+ if (IV.getTruncInst())
continue;
// A sequence of IR Casts has potentially been recorded for IV, which
@@ -697,8 +695,8 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
// replace it with the original IV. Note that only the final cast is
// expected to have users outside the cast-chain and the dead casts left
// over will be cleaned up later.
- ArrayRef<Instruction *> Casts = IV->getInductionDescriptor().getCastInsts();
- VPValue *FindMyCast = IV;
+ ArrayRef<Instruction *> Casts = IV.getInductionDescriptor().getCastInsts();
+ VPValue *FindMyCast = &IV;
for (Instruction *IRCast : reverse(Casts)) {
VPSingleDefRecipe *FoundUserCast = nullptr;
for (auto *U : FindMyCast->users()) {
@@ -713,8 +711,8 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
break;
FindMyCast = FoundUserCast;
}
- if (FindMyCast != IV)
- FindMyCast->replaceAllUsesWith(IV);
+ if (FindMyCast != &IV)
+ FindMyCast->replaceAllUsesWith(&IV);
}
}
@@ -790,9 +788,9 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
SmallVector<VPWidenInductionRecipe *> WideIVs;
- for (VPRecipeBase &Phi : HeaderVPBB->phis())
- if (auto *PhiR = dyn_cast<VPWidenInductionRecipe>(&Phi))
- WideIVs.push_back(PhiR);
+ for (VPWidenInductionRecipe &PhiR :
+ make_isa_range<VPWidenInductionRecipe>(HeaderVPBB->phis()))
+ WideIVs.push_back(&PhiR);
// Try to narrow wide and replicating recipes to uniform recipes, based on
// VPlan analysis.
@@ -1115,13 +1113,11 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
DenseMap<VPValue *, VPValue *> EndValues;
VPValue *ResumeTC =
Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
- for (auto &Phi : VectorRegion->getEntryBasicBlock()->phis()) {
- auto *WideIV = dyn_cast<VPWidenInductionRecipe>(&Phi);
- if (!WideIV)
- continue;
- if (VPValue *EndValue =
- tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, ResumeTC))
- EndValues[WideIV] = EndValue;
+ for (VPWidenInductionRecipe &WideIV : make_isa_range<VPWidenInductionRecipe>(
+ VectorRegion->getEntryBasicBlock()->phis())) {
+ if (VPValue *EndValue = tryToComputeEndValueForInduction(
+ &WideIV, VectorPHBuilder, ResumeTC))
+ EndValues[&WideIV] = EndValue;
}
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
@@ -1165,21 +1161,18 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
DenseMap<const SCEV *, VPValue *> SCEV2VPV;
- for (VPRecipeBase &R :
- make_early_inc_range(*Plan.getEntry()->getEntryBasicBlock())) {
- auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpR)
- continue;
-
- const auto &[V, Inserted] = SCEV2VPV.try_emplace(ExpR->getSCEV(), ExpR);
+ for (VPExpandSCEVRecipe &ExpR :
+ make_early_inc_range(make_isa_range<VPExpandSCEVRecipe>(
+ *Plan.getEntry()->getEntryBasicBlock()))) {
+ const auto &[V, Inserted] = SCEV2VPV.try_emplace(ExpR.getSCEV(), &ExpR);
if (Inserted)
continue;
- ExpR->replaceAllUsesWith(V->second);
- if (ExpR == Plan.getTripCount())
+ ExpR.replaceAllUsesWith(V->second);
+ if (&ExpR == Plan.getTripCount())
Plan.resetTripCount(V->second);
- ExpR->eraseFromParent();
+ ExpR.eraseFromParent();
}
}
@@ -1728,9 +1721,9 @@ void VPlanTransforms::combineRecipes(VPlan &Plan) {
PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(POT))
- for (VPRecipeBase &R : reverse(*VPBB))
- if (auto *Def = dyn_cast<VPSingleDefRecipe>(&R))
- Worklist.push_back(Def);
+ for (VPSingleDefRecipe &Def :
+ make_isa_range<VPSingleDefRecipe>(reverse(*VPBB)))
+ Worklist.push_back(&Def);
[[maybe_unused]] unsigned InitWorklistSize = Worklist.size();
@@ -1933,39 +1926,36 @@ static void removeCommonBlendMask(VPBlendRecipe *Blend) {
static void simplifyBlends(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *Blend = dyn_cast<VPBlendRecipe>(&R);
- if (!Blend)
- continue;
-
- removeCommonBlendMask(Blend);
+ for (VPBlendRecipe &Blend :
+ make_early_inc_range(make_isa_range<VPBlendRecipe>(*VPBB))) {
+ removeCommonBlendMask(&Blend);
// Try to remove redundant blend recipes.
SmallPtrSet<VPValue *, 4> UniqueValues;
- if (Blend->isNormalized() || !match(Blend->getMask(0), m_False()))
- UniqueValues.insert(Blend->getIncomingValue(0));
- for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
- if (!match(Blend->getMask(I), m_False()))
- UniqueValues.insert(Blend->getIncomingValue(I));
+ if (Blend.isNormalized() || !match(Blend.getMask(0), m_False()))
+ UniqueValues.insert(Blend.getIncomingValue(0));
+ for (unsigned I = 1; I != Blend.getNumIncomingValues(); ++I)
+ if (!match(Blend.getMask(I), m_False()))
+ UniqueValues.insert(Blend.getIncomingValue(I));
if (UniqueValues.size() == 1) {
- Blend->replaceAllUsesWith(*UniqueValues.begin());
- Blend->eraseFromParent();
+ Blend.replaceAllUsesWith(*UniqueValues.begin());
+ Blend.eraseFromParent();
continue;
}
- if (Blend->isNormalized())
+ if (Blend.isNormalized())
continue;
// Normalize the blend so its first incoming value is used as the initial
// value with the others blended into it.
unsigned StartIndex = 0;
- for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
+ for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
// If a value's mask is used only by the blend then is can be deadcoded.
// TODO: Find the most expensive mask that can be deadcoded, or a mask
// that's used by multiple blends where it can be removed from them all.
- VPValue *Mask = Blend->getMask(I);
+ VPValue *Mask = Blend.getMask(I);
if (Mask->hasOneUse() && !match(Mask, m_False())) {
StartIndex = I;
break;
@@ -1973,23 +1963,23 @@ static void simplifyBlends(VPlan &Plan) {
}
SmallVector<VPValue *, 4> OperandsWithMask;
- OperandsWithMask.push_back(Blend->getIncomingValue(StartIndex));
+ OperandsWithMask.push_back(Blend.getIncomingValue(StartIndex));
- for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
+ for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
if (I == StartIndex)
continue;
- OperandsWithMask.push_back(Blend->getIncomingValue(I));
- OperandsWithMask.push_back(Blend->getMask(I));
+ OperandsWithMask.push_back(Blend.getIncomingValue(I));
+ OperandsWithMask.push_back(Blend.getMask(I));
}
auto *NewBlend =
- new VPBlendRecipe(cast_or_null<PHINode>(Blend->getUnderlyingValue()),
- OperandsWithMask, *Blend, Blend->getDebugLoc());
- NewBlend->insertBefore(&R);
+ new VPBlendRecipe(cast_or_null<PHINode>(Blend.getUnderlyingValue()),
+ OperandsWithMask, Blend, Blend.getDebugLoc());
+ NewBlend->insertBefore(&Blend);
- VPValue *DeadMask = Blend->getMask(StartIndex);
- Blend->replaceAllUsesWith(NewBlend);
- Blend->eraseFromParent();
+ VPValue *DeadMask = Blend.getMask(StartIndex);
+ Blend.replaceAllUsesWith(NewBlend);
+ Blend.eraseFromParent();
vputils::recursivelyDeleteDeadRecipes(DeadMask);
/// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
@@ -2240,17 +2230,14 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
}
void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
- for (VPRecipeBase &R :
- Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
- auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!PhiR)
- continue;
- RecurKind RK = PhiR->getRecurrenceKind();
+ for (VPReductionPHIRecipe &PhiR : make_isa_range<VPReductionPHIRecipe>(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
+ RecurKind RK = PhiR.getRecurrenceKind();
if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
RK != RecurKind::AddChainWithSubs)
continue;
- for (VPUser *U : vputils::collectUsersRecursively(PhiR))
+ for (VPUser *U : vputils::collectUsersRecursively(&PhiR))
if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(U)) {
RecWithFlags->dropPoisonGeneratingFlags();
}
@@ -2791,17 +2778,15 @@ void VPlanTransforms::replaceSymbolicStrides(
RewriteMap[StrideV] = StrideExpr;
}
- for (VPRecipeBase &R : *Plan.getEntry()) {
- auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpSCEV)
- continue;
- const SCEV *ScevExpr = ExpSCEV->getSCEV();
+ for (VPExpandSCEVRecipe &ExpSCEV :
+ make_isa_range<VPExpandSCEVRecipe>(*Plan.getEntry())) {
+ const SCEV *ScevExpr = ExpSCEV.getSCEV();
auto *NewSCEV =
SCEVParameterRewriter::rewrite(ScevExpr, *PSE.getSE(), RewriteMap);
if (NewSCEV != ScevExpr) {
VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, NewSCEV);
- ExpSCEV->replaceAllUsesWith(NewExp);
- if (Plan.getTripCount() == ExpSCEV)
+ ExpSCEV.replaceAllUsesWith(NewExp);
+ if (Plan.getTripCount() == &ExpSCEV)
Plan.resetTripCount(NewExp);
}
}
@@ -3816,10 +3801,9 @@ void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
VFRange &Range) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (auto *Red = dyn_cast<VPReductionRecipe>(&R))
- tryToCreateAbstractReductionRecipe(Red, Ctx, Range);
- }
+ for (VPReductionRecipe &Red :
+ make_early_inc_range(make_isa_range<VPReductionRecipe>(*VPBB)))
+ tryToCreateAbstractReductionRecipe(&Red, Ctx, Range);
}
}
@@ -4399,11 +4383,9 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
return VF == ElementCount::getScalable(1);
};
- for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
- auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
- if (!FOR)
- continue;
-
+ for (VPFirstOrderRecurrencePHIRecipe &FOR :
+ make_isa_range<VPFirstOrderRecurrencePHIRecipe>(
+ VectorRegion->getEntryBasicBlock()->phis())) {
assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
"Cannot handle loops with uncountable early exits");
@@ -4411,7 +4393,7 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
// createHeaderPhiRecipes. All uses of FOR have already been replaced with
// RecurSplice there; only RecurSplice itself still references FOR.
auto *RecurSplice =
- findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(FOR);
+ findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(&FOR);
assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
// For VF vscale x 1, if vscale = 1, we are unable to extract the
@@ -5355,18 +5337,15 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
ChainsByPhi;
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
- for (VPRecipeBase &R : HeaderVPBB->phis()) {
- auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!RedPhiR)
- continue;
-
- if (auto Chains = getScaledReductions(RedPhiR))
- ChainsByPhi.try_emplace(RedPhiR, std::move(*Chains));
+ for (VPReductionPHIRecipe &RedPhiR :
+ make_isa_range<VPReductionPHIRecipe>(HeaderVPBB->phis())) {
+ if (auto Chains = getScaledReductions(&RedPhiR))
+ ChainsByPhi.try_emplace(&RedPhiR, std::move(*Chains));
else if (UsePartialReductionsByDefault &&
- (RedPhiR->getRecurrenceKind() == RecurKind::Add ||
- (RedPhiR->getRecurrenceKind() == RecurKind::FAdd &&
- !RedPhiR->isOrdered() && !RedPhiR->isInLoop())))
- UnorderedReductions.push_back(RedPhiR);
+ (RedPhiR.getRecurrenceKind() == RecurKind::Add ||
+ (RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
+ !RedPhiR.isOrdered() && !RedPhiR.isInLoop())))
+ UnorderedReductions.push_back(&RedPhiR);
}
// For general unordered reductions which aren't part of a candidate chain for
@@ -5527,12 +5506,11 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
for (VPBasicBlock *VPBB :
VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(
Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
- for (VPRecipeBase &R : *VPBB) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (VPI && VPI->getUnderlyingValue() &&
+ for (VPInstruction &VPI : make_isa_range<VPInstruction>(*VPBB)) {
+ if (VPI.getUnderlyingValue() &&
is_contained({Instruction::Load, Instruction::Store},
- VPI->getOpcode()))
- MemOps.push_back(VPI);
+ VPI.getOpcode()))
+ MemOps.push_back(&VPI);
}
}
@@ -5705,40 +5683,37 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(POT)) {
- for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (!VPI)
- continue;
-
- auto *I = cast_or_null<Instruction>(VPI->getUnderlyingValue());
+ for (VPInstruction &VPI :
+ make_early_inc_range(make_isa_range<VPInstruction>(reverse(*VPBB)))) {
+ auto *I = cast_or_null<Instruction>(VPI.getUnderlyingValue());
// Wouldn't be able to create a `VPReplicateRecipe` anyway.
if (!I)
continue;
// If executing other lanes produces side-effects we can't avoid them.
- if (VPI->mayHaveSideEffects())
+ if (VPI.mayHaveSideEffects())
continue;
// We want to drop the mask operand, verify we can safely do that.
- if (VPI->isMasked() && !VPI->isSafeToSpeculativelyExecute())
+ if (VPI.isMasked() && !VPI.isSafeToSpeculativelyExecute())
continue;
// Avoid rewriting IV increment as that interferes with
// `removeRedundantCanonicalIVs`.
- if (VPI->getOpcode() == Instruction::Add &&
- any_of(VPI->operands(), IsaPred<VPWidenIntOrFpInductionRecipe>))
+ if (VPI.getOpcode() == Instruction::Add &&
+ any_of(VPI.operands(), IsaPred<VPWidenIntOrFpInductionRecipe>))
continue;
// Other lanes are needed - can't drop them.
- if (!vputils::onlyFirstLaneUsed(VPI))
+ if (!vputils::onlyFirstLaneUsed(&VPI))
continue;
auto *Recipe = VPBuilder::createSingleScalarOp(
- VPI->getOpcode(), VPI->operandsWithoutMask(), /*Mask=*/nullptr, *VPI,
- *VPI, VPI->getDebugLoc(), I);
- Recipe->insertBefore(VPI);
- VPI->replaceAllUsesWith(Recipe);
- VPI->eraseFromParent();
+ VPI.getOpcode(), VPI.operandsWithoutMask(), /*Mask=*/nullptr, VPI,
+ VPI, VPI.getDebugLoc(), I);
+ Recipe->insertBefore(&VPI);
+ VPI.replaceAllUsesWith(Recipe);
+ VPI.eraseFromParent();
}
}
}
@@ -5858,21 +5833,20 @@ void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPCostContext &CostCtx) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (!VPI || !VPI->getUnderlyingValue() ||
- VPI->getOpcode() != Instruction::Call)
+ for (VPInstruction &VPI :
+ make_early_inc_range(make_isa_range<VPInstruction>(*VPBB))) {
+ if (!VPI.getUnderlyingValue() || VPI.getOpcode() != Instruction::Call)
continue;
- auto *CI = cast<CallInst>(VPI->getUnderlyingInstr());
- SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
- VPI->op_begin() + CI->arg_size());
+ auto *CI = cast<CallInst>(VPI.getUnderlyingInstr());
+ SmallVector<VPValue *, 4> Ops(VPI.op_begin(),
+ VPI.op_begin() + CI->arg_size());
CallWideningDecision Decision =
- decideCallWidening(*VPI, Ops, Range.Start, CostCtx);
+ decideCallWidening(VPI, Ops, Range.Start, CostCtx);
LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
- return Decision == decideCallWidening(*VPI, Ops, VF, CostCtx);
+ return Decision == decideCallWidening(VPI, Ops, VF, CostCtx);
},
Range);
@@ -5880,31 +5854,31 @@ void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
switch (Decision.Kind) {
case CallWideningDecision::KindTy::Intrinsic: {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &CostCtx.TLI);
- Type *ResultTy = VPI->getScalarType();
- Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
- *VPI, VPI->getDebugLoc());
+ Type *ResultTy = VPI.getScalarType();
+ Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, VPI,
+ VPI, VPI.getDebugLoc());
break;
}
case CallWideningDecision::KindTy::VectorVariant: {
// Masked variants take the mask as a trailing parameter, so they have
// one more parameter than the original call's arguments.
if (Decision.Variant->arg_size() > Ops.size()) {
- VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
+ VPValue *Mask = VPI.isMasked() ? VPI.getMask() : Plan.getTrue();
Ops.push_back(Mask);
}
- Ops.push_back(VPI->getOperand(VPI->getNumOperandsWithoutMask() - 1));
- Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, *VPI,
- *VPI, VPI->getDebugLoc());
+ Ops.push_back(VPI.getOperand(VPI.getNumOperandsWithoutMask() - 1));
+ Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, VPI, VPI,
+ VPI.getDebugLoc());
break;
}
case CallWideningDecision::KindTy::Scalarize:
- Replacement = RecipeBuilder.handleReplication(VPI, Range);
+ Replacement = RecipeBuilder.handleReplication(&VPI, Range);
break;
}
- Replacement->insertBefore(VPI);
- VPI->replaceAllUsesWith(Replacement);
- VPI->eraseFromParent();
+ Replacement->insertBefore(&VPI);
+ VPI.replaceAllUsesWith(Replacement);
+ VPI.eraseFromParent();
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 52a16debc7acf..1be9bdf6c01ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -508,13 +508,12 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF) {
auto Iter = vp_depth_first_deep(Plan.getEntry());
// Remove recipes that are redundant after unrolling.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *VPI = dyn_cast<VPInstruction>(&R);
- if (VPI &&
- VPI->getOpcode() == VPInstruction::CanonicalIVIncrementForPart &&
- VPI->getOperand(1) == &Plan.getVF()) {
- VPI->replaceAllUsesWith(VPI->getOperand(0));
- VPI->eraseFromParent();
+ for (VPInstruction &VPI :
+ make_early_inc_range(make_isa_range<VPInstruction>(*VPBB))) {
+ if (VPI.getOpcode() == VPInstruction::CanonicalIVIncrementForPart &&
+ VPI.getOperand(1) == &Plan.getVF()) {
+ VPI.replaceAllUsesWith(VPI.getOperand(0));
+ VPI.eraseFromParent();
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c440d37bef517..43fae60d87aea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1340,29 +1340,29 @@ void vputils::detail::pullOutPermutationsImpl(
function_ref<VPSingleDefRecipe *(VPSingleDefRecipe *X)> BuildPerm) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
- if (!Def || !isElementwise(Def))
+ for (VPSingleDefRecipe &Def :
+ make_early_inc_range(make_isa_range<VPSingleDefRecipe>(*VPBB))) {
+ if (!isElementwise(&Def))
continue;
// At least one of the ops must be a permutation.
- if (none_of(Def->operands(), MatchPerm))
+ if (none_of(Def.operands(), MatchPerm))
continue;
// All operands must be a single-use permutation or a live in (splat).
- if (!all_of(Def->operands(), [&MatchPerm](VPValue *Op) {
+ if (!all_of(Def.operands(), [&MatchPerm](VPValue *Op) {
return (Op->hasOneUse() && MatchPerm(Op)) || match(Op, m_LiveIn());
}))
continue;
// Remove the inner permutations.
- for (unsigned I = 0, E = Def->getNumOperands(); I != E; ++I)
- if (VPValue *X = MatchPerm(Def->getOperand(I)))
- Def->setOperand(I, X);
+ for (unsigned I = 0, E = Def.getNumOperands(); I != E; ++I)
+ if (VPValue *X = MatchPerm(Def.getOperand(I)))
+ Def.setOperand(I, X);
- VPSingleDefRecipe *Res = BuildPerm(Def);
- Res->insertAfter(Def);
- Def->replaceUsesWithIf(
+ VPSingleDefRecipe *Res = BuildPerm(&Def);
+ Res->insertAfter(&Def);
+ Def.replaceUsesWithIf(
Res, [&Res](VPUser &U, unsigned _) { return &U != Res; });
}
}
More information about the llvm-commits
mailing list