[llvm] [VPlan] Use VPPhiAccessors for VPIRPhi. (PR #129389)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Mar 1 08:12:20 PST 2025
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/129389
Use VPPhiAccessors for VPIRPhi.
Depends on https://github.com/llvm/llvm-project/pull/129388 and https://github.com/llvm/llvm-project/pull/129387 (both included in the PR).
>From c305e73d748277823b65c0ee5ddbee107e557f9b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 28 Feb 2025 14:03:45 +0000
Subject: [PATCH 1/3] [VPlan] Add new VPIRPhi overlay for VPIRInsts wrapping
phi nodes (NFC).
Add a new VPIRPhi subclass of VPIRInstruction, that purely serves as an
overlay, to provide more convenient checking (via directly doing
isa/dyn_cast/cast) and specialied execute/print implementations.
Both VPIRInstruction and VPIRPhi share the same VPDefID, and are
differentiated by the backing IR instruction.
This pattern could alos be used to provide more specialized interfaces for
some VPInstructions ocpodes, without introducing new, completely
spearate recipes. An example would be modeling VPWidenPHIRecipe &
VPScalarPHIRecip using VPInstructions opcodes and providing an interface
to retrieve incoming blocks and values through a VPInstruction subclass
similar to VPIRPhi.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 28 +++++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 65 ++++++++++++-------
.../Transforms/Vectorize/VPlanTransforms.cpp | 14 ++--
.../Transforms/Vectorize/VPlanVerifier.cpp | 6 +-
6 files changed, 89 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e2612698b6b0f..bfbd698b140ad 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9089,14 +9089,14 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
VPValue *OneVPV = Plan.getOrAddLiveIn(
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
- auto *ScalarPhiIRI = cast<VPIRInstruction>(&ScalarPhiR);
- auto *ScalarPhiI = dyn_cast<PHINode>(&ScalarPhiIRI->getInstruction());
- if (!ScalarPhiI)
+ auto *ScalarPhiIRI = dyn_cast<VPIRPhi>(&ScalarPhiR);
+ if (!ScalarPhiIRI)
break;
// TODO: Extract final value from induction recipe initially, optimize to
// pre-computed end value together in optimizeInductionExitUsers.
- auto *VectorPhiR = cast<VPHeaderPHIRecipe>(Builder.getRecipe(ScalarPhiI));
+ auto *VectorPhiR =
+ cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
@@ -9146,11 +9146,8 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
continue;
for (VPRecipeBase &R : *ExitVPBB) {
- auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
+ auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
if (!ExitIRI)
- continue;
- auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
- if (!ExitPhi)
break;
if (ExitVPBB->getSinglePredecessor() != Plan.getMiddleBlock()) {
assert(ExitIRI->getNumOperands() ==
@@ -9158,8 +9155,10 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
"early-exit must update exit values on construction");
continue;
}
+
+ PHINode &ExitPhi = ExitIRI->getIRPhi();
BasicBlock *ExitingBB = OrigLoop->getLoopLatch();
- Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
+ Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB);
VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
ExitIRI->addOperand(V);
if (V->isLiveIn())
@@ -10297,11 +10296,10 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
}
for (VPRecipeBase &R : make_early_inc_range(*MainPlan.getScalarHeader())) {
- auto *VPIRInst = cast<VPIRInstruction>(&R);
- auto *IRI = dyn_cast<PHINode>(&VPIRInst->getInstruction());
- if (!IRI)
+ auto *VPIRInst = dyn_cast<VPIRPhi>(&R);
+ if (!VPIRInst)
break;
- if (EpiWidenedPhis.contains(IRI))
+ if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
continue;
// There is no corresponding wide induction in the epilogue plan that would
// need a resume value. Remove the VPIRInst wrapping the scalar header phi
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 563784e4af924..8a7d5a904c46b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1295,7 +1295,7 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
auto *VPIRBB = createEmptyVPIRBasicBlock(IRBB);
for (Instruction &I :
make_range(IRBB->begin(), IRBB->getTerminator()->getIterator()))
- VPIRBB->appendRecipe(new VPIRInstruction(I));
+ VPIRBB->appendRecipe(VPIRInstruction::create(I));
return VPIRBB;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 70e684826ed2d..1c6099e6d8d9f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1032,16 +1032,19 @@ class VPInstruction : public VPRecipeWithIRFlags,
class VPIRInstruction : public VPRecipeBase {
Instruction &I;
-public:
+protected:
VPIRInstruction(Instruction &I)
: VPRecipeBase(VPDef::VPIRInstructionSC, ArrayRef<VPValue *>()), I(I) {}
+public:
~VPIRInstruction() override = default;
+ static VPIRInstruction *create(Instruction &I);
+
VP_CLASSOF_IMPL(VPDef::VPIRInstructionSC)
VPIRInstruction *clone() override {
- auto *R = new VPIRInstruction(I);
+ auto *R = create(I);
for (auto *Op : operands())
R->addOperand(Op);
return R;
@@ -1085,6 +1088,27 @@ class VPIRInstruction : public VPRecipeBase {
void extractLastLaneOfOperand(VPBuilder &Builder);
};
+/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
+/// cast/dyn_cast/isa and execute() implementation.
+struct VPIRPhi : public VPIRInstruction {
+ VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {}
+
+ static inline bool classof(const VPRecipeBase *U) {
+ auto *R = dyn_cast<VPIRInstruction>(U);
+ return R && isa<PHINode>(R->getInstruction());
+ }
+
+ PHINode &getIRPhi() { return cast<PHINode>(getInstruction()); }
+
+ void execute(VPTransformState &State) override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e9f50e88867b2..0a315e85f33ec 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -961,30 +961,15 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPIRInstruction::execute(VPTransformState &State) {
- assert((isa<PHINode>(&I) || getNumOperands() == 0) &&
- "Only PHINodes can have extra operands");
- for (const auto &[Idx, Op] : enumerate(operands())) {
- VPValue *ExitValue = Op;
- auto Lane = vputils::isUniformAfterVectorization(ExitValue)
- ? VPLane::getFirstLane()
- : VPLane::getLastLaneForVF(State.VF);
- VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
- auto *PredVPBB = Pred->getExitingBasicBlock();
- BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
- // Set insertion point in PredBB in case an extract needs to be generated.
- // TODO: Model extracts explicitly.
- State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
- Value *V = State.get(ExitValue, VPLane(Lane));
- auto *Phi = cast<PHINode>(&I);
- // If there is no existing block for PredBB in the phi, add a new incoming
- // value. Otherwise update the existing incoming value for PredBB.
- if (Phi->getBasicBlockIndex(PredBB) == -1)
- Phi->addIncoming(V, PredBB);
- else
- Phi->setIncomingValueForBlock(PredBB, V);
- }
+VPIRInstruction *VPIRInstruction ::create(Instruction &I) {
+ if (auto *Phi = dyn_cast<PHINode>(&I))
+ return new VPIRPhi(*Phi);
+ return new VPIRInstruction(I);
+}
+void VPIRInstruction::execute(VPTransformState &State) {
+ assert(!isa<VPIRPhi>(this) && getNumOperands() == 0 &&
+ "PHINodes must be handled by VPIRPhi");
// Advance the insert point after the wrapped IR instruction. This allows
// interleaving VPIRInstructions and other recipes.
State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));
@@ -1017,6 +1002,40 @@ void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) {
void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "IR " << I;
+}
+#endif
+
+void VPIRPhi::execute(VPTransformState &State) {
+ PHINode *Phi = &getIRPhi();
+ for (const auto &[Idx, Op] : enumerate(operands())) {
+ VPValue *ExitValue = Op;
+ auto Lane = vputils::isUniformAfterVectorization(ExitValue)
+ ? VPLane::getFirstLane()
+ : VPLane::getLastLaneForVF(State.VF);
+ VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
+ auto *PredVPBB = Pred->getExitingBasicBlock();
+ BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
+ // Set insertion point in PredBB in case an extract needs to be generated.
+ // TODO: Model extracts explicitly.
+ State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
+ Value *V = State.get(ExitValue, VPLane(Lane));
+ // If there is no existing block for PredBB in the phi, add a new incoming
+ // value. Otherwise update the existing incoming value for PredBB.
+ if (Phi->getBasicBlockIndex(PredBB) == -1)
+ Phi->addIncoming(V, PredBB);
+ else
+ Phi->setIncomingValueForBlock(PredBB, V);
+ }
+
+ // Advance the insert point after the wrapped IR instruction. This allows
+ // interleaving VPIRInstructions and other recipes.
+ State.Builder.SetInsertPoint(Phi->getParent(), std::next(Phi->getIterator()));
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ VPIRInstruction::print(O, Indent, SlotTracker);
if (getNumOperands() != 0) {
O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b09933cd0e186..8627571e66b6c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -749,8 +749,8 @@ void VPlanTransforms::optimizeInductionExitUsers(
VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
VPBuilder B(Plan.getMiddleBlock()->getTerminator());
for (VPRecipeBase &R : *ExitVPBB) {
- auto *ExitIRI = cast<VPIRInstruction>(&R);
- if (!isa<PHINode>(ExitIRI->getInstruction()))
+ auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
+ if (!ExitIRI)
break;
VPValue *Incoming;
@@ -2088,20 +2088,20 @@ void VPlanTransforms::handleUncountableEarlyExit(
VPBuilder MiddleBuilder(NewMiddle);
VPBuilder EarlyExitB(VectorEarlyExitVPBB);
for (VPRecipeBase &R : *VPEarlyExitBlock) {
- auto *ExitIRI = cast<VPIRInstruction>(&R);
- auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
- if (!ExitPhi)
+ auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
+ if (!ExitIRI)
break;
+ PHINode &ExitPhi = ExitIRI->getIRPhi();
VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn(
- ExitPhi->getIncomingValueForBlock(UncountableExitingBlock));
+ ExitPhi.getIncomingValueForBlock(UncountableExitingBlock));
if (OrigLoop->getUniqueExitBlock()) {
// If there's a unique exit block, VPEarlyExitBlock has 2 predecessors
// (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB
// which is coming from the original latch.
VPValue *IncomingFromLatch = RecipeBuilder.getVPValueOrAddLiveIn(
- ExitPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
+ ExitPhi.getIncomingValueForBlock(OrigLoop->getLoopLatch()));
ExitIRI->addOperand(IncomingFromLatch);
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index a4b309d6dcd9f..f98323b77c708 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -204,10 +204,8 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
for (const VPUser *U : V->users()) {
auto *UI = cast<VPRecipeBase>(U);
// TODO: check dominance of incoming values for phis properly.
- if (!UI ||
- isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
- (isa<VPIRInstruction>(UI) &&
- isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())))
+ if (!UI || isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe,
+ VPIRPhi>(UI))
continue;
// If the user is in the same block, check it comes after R in the
>From 5df36e46c046f5e854b3010763a1c47acf567784 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 27 Feb 2025 21:49:14 +0000
Subject: [PATCH 2/3] [VPlan] Add VPPhiAccessors to provide interface for phi
recipes (NFC)
Add a VPPhiAccessors class to provide interfaces to access incoming
values and blocks, with corresponding iterators.
The first user is VPWidenPhiRecipe, with the other phi-like recipes
following soon.
This will also be used to verify def-use chains where users are phi-like
recipes, simplifying https://github.com/llvm/llvm-project/pull/124838.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 65 +++++++++++++++++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 46 +++++++------
3 files changed, 87 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bfbd698b140ad..781e7295d4bb2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3069,11 +3069,9 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
// Make sure the builder has a valid insert point.
Builder.SetInsertPoint(NewPhi);
- for (unsigned Idx = 0; Idx < VPPhi->getNumOperands(); ++Idx) {
- VPValue *Inc = VPPhi->getIncomingValue(Idx);
- VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
+
+ for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
- }
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1c6099e6d8d9f..79afb9f1799bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1088,6 +1088,62 @@ class VPIRInstruction : public VPRecipeBase {
void extractLastLaneOfOperand(VPBuilder &Builder);
};
+/// Helper type to provide functions to access incoming values and blocks for
+/// phi-like recipes. RecipeTy must be a sub-class of VPRecipeBase.
+template <typename RecipeTy> class VPPhiAccessors {
+ /// Return a VPRecipeBase* to the current object.
+ const VPRecipeBase *getAsRecipe() const {
+ return static_cast<const RecipeTy *>(this);
+ }
+
+public:
+ /// Returns the \p I th incoming VPValue.
+ VPValue *getIncomingValue(unsigned I) const {
+ return getAsRecipe()->getOperand(I);
+ }
+
+ /// Returns an interator range over the incoming values
+ VPUser::const_operand_range incoming_values() const {
+ return getAsRecipe()->operands();
+ }
+
+ /// Returns the \p I th incoming block.
+ const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
+
+ using const_incoming_block_iterator =
+ mapped_iterator<detail::index_iterator,
+ std::function<const VPBasicBlock *(size_t)>>;
+ using const_incoming_blocks_range =
+ iterator_range<const_incoming_block_iterator>;
+
+ const_incoming_block_iterator incoming_block_begin() const {
+ return const_incoming_block_iterator(
+ detail::index_iterator(0),
+ [this](size_t Idx) { return getIncomingBlock(Idx); });
+ }
+ const_incoming_block_iterator incoming_block_end() const {
+ return const_incoming_block_iterator(
+ detail::index_iterator(getAsRecipe()->getVPDefID() ==
+ VPDef::VPWidenIntOrFpInductionSC
+ ? 2
+ : getAsRecipe()->getNumOperands()),
+ [this](size_t Idx) { return getIncomingBlock(Idx); });
+ }
+
+ /// Returns an iterator range over the incoming blocks.
+ const_incoming_blocks_range incoming_blocks() const {
+ return make_range(incoming_block_begin(), incoming_block_end());
+ }
+
+ /// Returns an iterator range over pairs of incoming values and corrsponding
+ /// incoming blocks.
+ detail::zippy<llvm::detail::zip_shortest, VPUser::const_operand_range,
+ const_incoming_blocks_range>
+ incoming_values_and_blocks() const {
+ return zip(incoming_values(), incoming_blocks());
+ }
+};
+
/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
/// cast/dyn_cast/isa and execute() implementation.
struct VPIRPhi : public VPIRInstruction {
@@ -1968,7 +2024,8 @@ class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
/// recipe is placed in an entry block to a (non-replicate) region, it must have
/// exactly 2 incoming values, the first from the predecessor of the region and
/// the second from the exiting block of the region.
-class VPWidenPHIRecipe : public VPSingleDefRecipe {
+class VPWidenPHIRecipe : public VPSingleDefRecipe,
+ public VPPhiAccessors<VPWidenPHIRecipe> {
public:
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and
/// debug location \p DL.
@@ -1994,12 +2051,6 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe {
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
-
- /// Returns the \p I th incoming VPBasicBlock.
- VPBasicBlock *getIncomingBlock(unsigned I);
-
- /// Returns the \p I th incoming VPValue.
- VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
};
/// A recipe for handling first-order recurrence phis. The start value is the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 0a315e85f33ec..cf84692745274 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1005,6 +1005,29 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
}
#endif
+static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R,
+ unsigned I) {
+ const VPBasicBlock *Parent = R->getParent();
+ const VPBlockBase *Pred = nullptr;
+ if (Parent->getNumPredecessors() > 0) {
+ Pred = Parent->getPredecessors()[I];
+ } else {
+ auto *Region = Parent->getParent();
+ assert(Region && !Region->isReplicator() && Region->getEntry() == Parent &&
+ "must be in the entry block of a non-replicate region");
+ assert(
+ I < 2 &&
+ (R->getNumOperands() == 2 || isa<VPWidenIntOrFpInductionRecipe>(R)) &&
+ "when placed in an entry block, only 2 incoming blocks are available");
+
+ // I == 0 selects the predecessor of the region, I == 1 selects the region
+ // itself whose exiting block feeds the phi across the backedge.
+ Pred = I == 0 ? Region->getSinglePredecessor() : Region;
+ }
+
+ return Pred->getExitingBasicBlock();
+}
+
void VPIRPhi::execute(VPTransformState &State) {
PHINode *Phi = &getIRPhi();
for (const auto &[Idx, Op] : enumerate(operands())) {
@@ -3599,25 +3622,10 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
-VPBasicBlock *VPWidenPHIRecipe::getIncomingBlock(unsigned I) {
- VPBasicBlock *Parent = getParent();
- VPBlockBase *Pred = nullptr;
- if (Parent->getNumPredecessors() > 0) {
- Pred = Parent->getPredecessors()[I];
- } else {
- auto *Region = Parent->getParent();
- assert(Region && !Region->isReplicator() && Region->getEntry() == Parent &&
- "must be in the entry block of a non-replicate region");
- assert(
- I < 2 && getNumOperands() == 2 &&
- "when placed in an entry block, only 2 incoming blocks are available");
-
- // I == 0 selects the predecessor of the region, I == 1 selects the region
- // itself whose exiting block feeds the phi across the backedge.
- Pred = I == 0 ? Region->getSinglePredecessor() : Region;
- }
-
- return Pred->getExitingBasicBlock();
+template <>
+const VPBasicBlock *
+VPPhiAccessors<VPWidenPHIRecipe>::getIncomingBlock(unsigned Idx) const {
+ return getIncomingBlockForRecipe(getAsRecipe(), Idx);
}
void VPWidenPHIRecipe::execute(VPTransformState &State) {
>From 4939125090c68a1d3bcc6915f802724ad7fef36c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 1 Mar 2025 15:19:14 +0000
Subject: [PATCH 3/3] [VPlan] Use VPPhiAccessors for VPIRPhi.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 24 ++++++++++---------
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 79afb9f1799bf..ed63a53555af6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1146,7 +1146,7 @@ template <typename RecipeTy> class VPPhiAccessors {
/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
/// cast/dyn_cast/isa and execute() implementation.
-struct VPIRPhi : public VPIRInstruction {
+struct VPIRPhi : public VPIRInstruction, public VPPhiAccessors<VPIRPhi> {
VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {}
static inline bool classof(const VPRecipeBase *U) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cf84692745274..68b04fe9dcbe0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1028,16 +1028,19 @@ static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R,
return Pred->getExitingBasicBlock();
}
+template <>
+const VPBasicBlock *
+VPPhiAccessors<VPIRPhi>::getIncomingBlock(unsigned Idx) const {
+ return getIncomingBlockForRecipe(getAsRecipe(), Idx);
+}
+
void VPIRPhi::execute(VPTransformState &State) {
PHINode *Phi = &getIRPhi();
- for (const auto &[Idx, Op] : enumerate(operands())) {
- VPValue *ExitValue = Op;
+ for (const auto &[ExitValue, IncVPBB] : incoming_values_and_blocks()) {
auto Lane = vputils::isUniformAfterVectorization(ExitValue)
? VPLane::getFirstLane()
: VPLane::getLastLaneForVF(State.VF);
- VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
- auto *PredVPBB = Pred->getExitingBasicBlock();
- BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
+ BasicBlock *PredBB = State.CFG.VPBB2IRBB[IncVPBB];
// Set insertion point in PredBB in case an extract needs to be generated.
// TODO: Model extracts explicitly.
State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
@@ -1062,12 +1065,11 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
if (getNumOperands() != 0) {
O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";
- interleaveComma(
- enumerate(operands()), O, [this, &O, &SlotTracker](auto Op) {
- Op.value()->printAsOperand(O, SlotTracker);
- O << " from ";
- getParent()->getPredecessors()[Op.index()]->printAsOperand(O);
- });
+ interleaveComma(incoming_values_and_blocks(), O, [&O, &SlotTracker](auto Op) {
+ std::get<0>(Op)->printAsOperand(O, SlotTracker);
+ O << " from ";
+ std::get<1>(Op)->printAsOperand(O);
+ });
O << ")";
}
}
More information about the llvm-commits
mailing list