[llvm] 7b75db5 - [VPlan] Add new VPIRPhi overlay for VPIRInsts wrapping phi nodes (NFC). (#129387)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 28 01:43:49 PDT 2025
Author: Florian Hahn
Date: 2025-03-28T08:43:46Z
New Revision: 7b75db5755e63ddefa35101cf6a1179de560d312
URL: https://github.com/llvm/llvm-project/commit/7b75db5755e63ddefa35101cf6a1179de560d312
DIFF: https://github.com/llvm/llvm-project/commit/7b75db5755e63ddefa35101cf6a1179de560d312.diff
LOG: [VPlan] Add new VPIRPhi overlay for VPIRInsts wrapping phi nodes (NFC). (#129387)
Add a new VPIRPhi subclass of VPIRInstruction, that purely serves as an
overlay, to provide more convenient checking (via directly doing
isa/dyn_cast/cast) and specialied execute/print implementations.
Both VPIRInstruction and VPIRPhi share the same VPDefID, and are
differentiated by the backing IR instruction.
This pattern could alos be used to provide more specialized interfaces
for some VPInstructions ocpodes, without introducing new, completely
spearate recipes. An example would be modeling VPWidenPHIRecipe &
VPScalarPHIRecip using VPInstructions opcodes and providing an interface
to retrieve incoming blocks and values through a VPInstruction subclass
similar to VPIRPhi.
PR: https://github.com/llvm/llvm-project/pull/129387
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a86815ff65cfe..667529c5c3296 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9079,14 +9079,14 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
VPValue *OneVPV = Plan.getOrAddLiveIn(
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
- auto *ScalarPhiIRI = cast<VPIRInstruction>(&ScalarPhiR);
- auto *ScalarPhiI = dyn_cast<PHINode>(&ScalarPhiIRI->getInstruction());
- if (!ScalarPhiI)
+ auto *ScalarPhiIRI = dyn_cast<VPIRPhi>(&ScalarPhiR);
+ if (!ScalarPhiIRI)
break;
// TODO: Extract final value from induction recipe initially, optimize to
// pre-computed end value together in optimizeInductionExitUsers.
- auto *VectorPhiR = cast<VPHeaderPHIRecipe>(Builder.getRecipe(ScalarPhiI));
+ auto *VectorPhiR =
+ cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
@@ -9136,11 +9136,8 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
continue;
for (VPRecipeBase &R : *ExitVPBB) {
- auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
+ auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
if (!ExitIRI)
- continue;
- auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
- if (!ExitPhi)
break;
if (ExitVPBB->getSinglePredecessor() != Plan.getMiddleBlock()) {
assert(ExitIRI->getNumOperands() ==
@@ -9148,8 +9145,10 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
"early-exit must update exit values on construction");
continue;
}
+
+ PHINode &ExitPhi = ExitIRI->getIRPhi();
BasicBlock *ExitingBB = OrigLoop->getLoopLatch();
- Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
+ Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB);
VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue);
ExitIRI->addOperand(V);
if (V->isLiveIn())
@@ -10347,11 +10346,10 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
}
for (VPRecipeBase &R : make_early_inc_range(*MainPlan.getScalarHeader())) {
- auto *VPIRInst = cast<VPIRInstruction>(&R);
- auto *IRI = dyn_cast<PHINode>(&VPIRInst->getInstruction());
- if (!IRI)
+ auto *VPIRInst = dyn_cast<VPIRPhi>(&R);
+ if (!VPIRInst)
break;
- if (EpiWidenedPhis.contains(IRI))
+ if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
continue;
// There is no corresponding wide induction in the epilogue plan that would
// need a resume value. Remove the VPIRInst wrapping the scalar header phi
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f6293fa19b7db..8b53c559f6533 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1225,7 +1225,7 @@ VPIRBasicBlock *VPlan::createVPIRBasicBlock(BasicBlock *IRBB) {
auto *VPIRBB = createEmptyVPIRBasicBlock(IRBB);
for (Instruction &I :
make_range(IRBB->begin(), IRBB->getTerminator()->getIterator()))
- VPIRBB->appendRecipe(new VPIRInstruction(I));
+ VPIRBB->appendRecipe(VPIRInstruction::create(I));
return VPIRBB;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f9af99704d2f9..c6637ba0833d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1026,22 +1026,28 @@ class VPInstruction : public VPRecipeWithIRFlags,
};
/// A recipe to wrap on original IR instruction not to be modified during
-/// execution, execept for PHIs. For PHIs, a single VPValue operand is allowed,
-/// and it is used to add a new incoming value for the single predecessor VPBB.
+/// execution, except for PHIs. PHIs are modeled via the VPIRPhi subclass.
/// Expect PHIs, VPIRInstructions cannot have any operands.
class VPIRInstruction : public VPRecipeBase {
Instruction &I;
-public:
+protected:
+ /// VPIRInstruction::create() should be used to create VPIRInstructions, as
+ /// subclasses may need to be created, e.g. VPIRPhi.
VPIRInstruction(Instruction &I)
: VPRecipeBase(VPDef::VPIRInstructionSC, ArrayRef<VPValue *>()), I(I) {}
+public:
~VPIRInstruction() override = default;
+ /// Create a new VPIRPhi for \p \I, if it is a PHINode, otherwise create a
+ /// VPIRInstruction.
+ static VPIRInstruction *create(Instruction &I);
+
VP_CLASSOF_IMPL(VPDef::VPIRInstructionSC)
VPIRInstruction *clone() override {
- auto *R = new VPIRInstruction(I);
+ auto *R = create(I);
for (auto *Op : operands())
R->addOperand(Op);
return R;
@@ -1085,6 +1091,29 @@ class VPIRInstruction : public VPRecipeBase {
void extractLastLaneOfOperand(VPBuilder &Builder);
};
+/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
+/// cast/dyn_cast/isa and execute() implementation. A single VPValue operand is
+/// allowed, and it is used to add a new incoming value for the single
+/// predecessor VPBB.
+struct VPIRPhi : public VPIRInstruction {
+ VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {}
+
+ static inline bool classof(const VPRecipeBase *U) {
+ auto *R = dyn_cast<VPIRInstruction>(U);
+ return R && isa<PHINode>(R->getInstruction());
+ }
+
+ PHINode &getIRPhi() { return cast<PHINode>(getInstruction()); }
+
+ void execute(VPTransformState &State) override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9a9f88c19c5e0..f7760aa3c8eed 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1064,30 +1064,15 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPIRInstruction::execute(VPTransformState &State) {
- assert((isa<PHINode>(&I) || getNumOperands() == 0) &&
- "Only PHINodes can have extra operands");
- for (const auto &[Idx, Op] : enumerate(operands())) {
- VPValue *ExitValue = Op;
- auto Lane = vputils::isUniformAfterVectorization(ExitValue)
- ? VPLane::getFirstLane()
- : VPLane::getLastLaneForVF(State.VF);
- VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
- auto *PredVPBB = Pred->getExitingBasicBlock();
- BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
- // Set insertion point in PredBB in case an extract needs to be generated.
- // TODO: Model extracts explicitly.
- State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
- Value *V = State.get(ExitValue, VPLane(Lane));
- auto *Phi = cast<PHINode>(&I);
- // If there is no existing block for PredBB in the phi, add a new incoming
- // value. Otherwise update the existing incoming value for PredBB.
- if (Phi->getBasicBlockIndex(PredBB) == -1)
- Phi->addIncoming(V, PredBB);
- else
- Phi->setIncomingValueForBlock(PredBB, V);
- }
+VPIRInstruction *VPIRInstruction ::create(Instruction &I) {
+ if (auto *Phi = dyn_cast<PHINode>(&I))
+ return new VPIRPhi(*Phi);
+ return new VPIRInstruction(I);
+}
+void VPIRInstruction::execute(VPTransformState &State) {
+ assert(!isa<VPIRPhi>(this) && getNumOperands() == 0 &&
+ "PHINodes must be handled by VPIRPhi");
// Advance the insert point after the wrapped IR instruction. This allows
// interleaving VPIRInstructions and other recipes.
State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));
@@ -1120,6 +1105,40 @@ void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) {
void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "IR " << I;
+}
+#endif
+
+void VPIRPhi::execute(VPTransformState &State) {
+ PHINode *Phi = &getIRPhi();
+ for (const auto &[Idx, Op] : enumerate(operands())) {
+ VPValue *ExitValue = Op;
+ auto Lane = vputils::isUniformAfterVectorization(ExitValue)
+ ? VPLane::getFirstLane()
+ : VPLane::getLastLaneForVF(State.VF);
+ VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
+ auto *PredVPBB = Pred->getExitingBasicBlock();
+ BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
+ // Set insertion point in PredBB in case an extract needs to be generated.
+ // TODO: Model extracts explicitly.
+ State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());
+ Value *V = State.get(ExitValue, VPLane(Lane));
+ // If there is no existing block for PredBB in the phi, add a new incoming
+ // value. Otherwise update the existing incoming value for PredBB.
+ if (Phi->getBasicBlockIndex(PredBB) == -1)
+ Phi->addIncoming(V, PredBB);
+ else
+ Phi->setIncomingValueForBlock(PredBB, V);
+ }
+
+ // Advance the insert point after the wrapped IR instruction. This allows
+ // interleaving VPIRInstructions and other recipes.
+ State.Builder.SetInsertPoint(Phi->getParent(), std::next(Phi->getIterator()));
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ VPIRInstruction::print(O, Indent, SlotTracker);
if (getNumOperands() != 0) {
O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a445e7d867344..a2e6182cf16e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -861,8 +861,8 @@ void VPlanTransforms::optimizeInductionExitUsers(
VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
for (VPRecipeBase &R : *ExitVPBB) {
- auto *ExitIRI = cast<VPIRInstruction>(&R);
- if (!isa<PHINode>(ExitIRI->getInstruction()))
+ auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
+ if (!ExitIRI)
break;
for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
@@ -2221,20 +2221,20 @@ void VPlanTransforms::handleUncountableEarlyExit(
VPBuilder MiddleBuilder(NewMiddle);
VPBuilder EarlyExitB(VectorEarlyExitVPBB);
for (VPRecipeBase &R : *VPEarlyExitBlock) {
- auto *ExitIRI = cast<VPIRInstruction>(&R);
- auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
- if (!ExitPhi)
+ auto *ExitIRI = dyn_cast<VPIRPhi>(&R);
+ if (!ExitIRI)
break;
+ PHINode &ExitPhi = ExitIRI->getIRPhi();
VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn(
- ExitPhi->getIncomingValueForBlock(UncountableExitingBlock));
+ ExitPhi.getIncomingValueForBlock(UncountableExitingBlock));
if (OrigLoop->getUniqueExitBlock()) {
// If there's a unique exit block, VPEarlyExitBlock has 2 predecessors
// (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB
// which is coming from the original latch.
VPValue *IncomingFromLatch = RecipeBuilder.getVPValueOrAddLiveIn(
- ExitPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
+ ExitPhi.getIncomingValueForBlock(OrigLoop->getLoopLatch()));
ExitIRI->addOperand(IncomingFromLatch);
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 6fe131879b1a2..f7fa659ba6a8a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -209,9 +209,8 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
auto *UI = cast<VPRecipeBase>(U);
// TODO: check dominance of incoming values for phis properly.
if (!UI ||
- isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
- (isa<VPIRInstruction>(UI) &&
- isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())) ||
+ isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe,
+ VPIRPhi>(UI) ||
(isa<VPInstruction>(UI) &&
cast<VPInstruction>(UI)->getOpcode() == Instruction::PHI))
continue;
More information about the llvm-commits
mailing list