[llvm] [VPlan] Verify dominance for incoming values of phi-like recipes. (PR #124838)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun May 11 11:12:40 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/124838
>From b357af496024fe12bc56a39b89dbff036b65b952 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 27 Feb 2025 21:49:14 +0000
Subject: [PATCH 1/7] [VPlan] Add VPPhiAccessors to provide interface for phi
recipes (NFC)
Add a VPPhiAccessors class to provide interfaces to access incoming
values and blocks, with corresponding iterators.
The first user is VPWidenPhiRecipe, with the other phi-like recipes
following soon.
This will also be used to verify def-use chains where users are phi-like
recipes, simplifying https://github.com/llvm/llvm-project/pull/124838.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 65 +++++++++++++++++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 46 +++++++------
3 files changed, 87 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e2612698b6b0f..98ff1a4cdc449 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3069,11 +3069,9 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
// Make sure the builder has a valid insert point.
Builder.SetInsertPoint(NewPhi);
- for (unsigned Idx = 0; Idx < VPPhi->getNumOperands(); ++Idx) {
- VPValue *Inc = VPPhi->getIncomingValue(Idx);
- VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
+
+ for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
- }
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 70e684826ed2d..4d5f2014a40dc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1085,6 +1085,62 @@ class VPIRInstruction : public VPRecipeBase {
void extractLastLaneOfOperand(VPBuilder &Builder);
};
+/// Helper type to provide functions to access incoming values and blocks for
+/// phi-like recipes. RecipeTy must be a sub-class of VPRecipeBase.
+template <typename RecipeTy> class VPPhiAccessors {
+ /// Return a VPRecipeBase* to the current object.
+ const VPRecipeBase *getAsRecipe() const {
+ return static_cast<const RecipeTy *>(this);
+ }
+
+public:
+ /// Returns the \p I th incoming VPValue.
+ VPValue *getIncomingValue(unsigned I) const {
+ return getAsRecipe()->getOperand(I);
+ }
+
+ /// Returns an interator range over the incoming values
+ VPUser::const_operand_range incoming_values() const {
+ return getAsRecipe()->operands();
+ }
+
+ /// Returns the \p I th incoming block.
+ const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
+
+ using const_incoming_block_iterator =
+ mapped_iterator<detail::index_iterator,
+ std::function<const VPBasicBlock *(size_t)>>;
+ using const_incoming_blocks_range =
+ iterator_range<const_incoming_block_iterator>;
+
+ const_incoming_block_iterator incoming_block_begin() const {
+ return const_incoming_block_iterator(
+ detail::index_iterator(0),
+ [this](size_t Idx) { return getIncomingBlock(Idx); });
+ }
+ const_incoming_block_iterator incoming_block_end() const {
+ return const_incoming_block_iterator(
+ detail::index_iterator(getAsRecipe()->getVPDefID() ==
+ VPDef::VPWidenIntOrFpInductionSC
+ ? 2
+ : getAsRecipe()->getNumOperands()),
+ [this](size_t Idx) { return getIncomingBlock(Idx); });
+ }
+
+ /// Returns an iterator range over the incoming blocks.
+ const_incoming_blocks_range incoming_blocks() const {
+ return make_range(incoming_block_begin(), incoming_block_end());
+ }
+
+ /// Returns an iterator range over pairs of incoming values and corrsponding
+ /// incoming blocks.
+ detail::zippy<llvm::detail::zip_shortest, VPUser::const_operand_range,
+ const_incoming_blocks_range>
+ incoming_values_and_blocks() const {
+ return zip(incoming_values(), incoming_blocks());
+ }
+};
+
/// VPWidenRecipe is a recipe for producing a widened instruction using the
/// opcode and operands of the recipe. This recipe covers most of the
/// traditional vectorization cases where each recipe transforms into a
@@ -1944,7 +2000,8 @@ class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
/// recipe is placed in an entry block to a (non-replicate) region, it must have
/// exactly 2 incoming values, the first from the predecessor of the region and
/// the second from the exiting block of the region.
-class VPWidenPHIRecipe : public VPSingleDefRecipe {
+class VPWidenPHIRecipe : public VPSingleDefRecipe,
+ public VPPhiAccessors<VPWidenPHIRecipe> {
public:
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and
/// debug location \p DL.
@@ -1970,12 +2027,6 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe {
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
-
- /// Returns the \p I th incoming VPBasicBlock.
- VPBasicBlock *getIncomingBlock(unsigned I);
-
- /// Returns the \p I th incoming VPValue.
- VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
};
/// A recipe for handling first-order recurrence phis. The start value is the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e9f50e88867b2..efac05785203c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1031,6 +1031,29 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
}
#endif
+static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R,
+ unsigned I) {
+ const VPBasicBlock *Parent = R->getParent();
+ const VPBlockBase *Pred = nullptr;
+ if (Parent->getNumPredecessors() > 0) {
+ Pred = Parent->getPredecessors()[I];
+ } else {
+ auto *Region = Parent->getParent();
+ assert(Region && !Region->isReplicator() && Region->getEntry() == Parent &&
+ "must be in the entry block of a non-replicate region");
+ assert(
+ I < 2 &&
+ (R->getNumOperands() == 2 || isa<VPWidenIntOrFpInductionRecipe>(R)) &&
+ "when placed in an entry block, only 2 incoming blocks are available");
+
+ // I == 0 selects the predecessor of the region, I == 1 selects the region
+ // itself whose exiting block feeds the phi across the backedge.
+ Pred = I == 0 ? Region->getSinglePredecessor() : Region;
+ }
+
+ return Pred->getExitingBasicBlock();
+}
+
void VPWidenCallRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
State.setDebugLocFrom(getDebugLoc());
@@ -3580,25 +3603,10 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
-VPBasicBlock *VPWidenPHIRecipe::getIncomingBlock(unsigned I) {
- VPBasicBlock *Parent = getParent();
- VPBlockBase *Pred = nullptr;
- if (Parent->getNumPredecessors() > 0) {
- Pred = Parent->getPredecessors()[I];
- } else {
- auto *Region = Parent->getParent();
- assert(Region && !Region->isReplicator() && Region->getEntry() == Parent &&
- "must be in the entry block of a non-replicate region");
- assert(
- I < 2 && getNumOperands() == 2 &&
- "when placed in an entry block, only 2 incoming blocks are available");
-
- // I == 0 selects the predecessor of the region, I == 1 selects the region
- // itself whose exiting block feeds the phi across the backedge.
- Pred = I == 0 ? Region->getSinglePredecessor() : Region;
- }
-
- return Pred->getExitingBasicBlock();
+template <>
+const VPBasicBlock *
+VPPhiAccessors<VPWidenPHIRecipe>::getIncomingBlock(unsigned Idx) const {
+ return getIncomingBlockForRecipe(getAsRecipe(), Idx);
}
void VPWidenPHIRecipe::execute(VPTransformState &State) {
>From c61233f4096202789ebeaa11061d70bb53cdd2af Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 13 Mar 2025 16:25:03 +0000
Subject: [PATCH 2/7] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlan.h | 15 ++++++---------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 17 ++++++++++-------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1915b90675193..63d946a0dede9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1095,9 +1095,9 @@ template <typename RecipeTy> class VPPhiAccessors {
}
public:
- /// Returns the \p I th incoming VPValue.
- VPValue *getIncomingValue(unsigned I) const {
- return getAsRecipe()->getOperand(I);
+ /// Returns the incoming VPValue with index \p Idx.
+ VPValue *getIncomingValue(unsigned Idx) const {
+ return getAsRecipe()->getOperand(Idx);
}
/// Returns an interator range over the incoming values
@@ -1105,7 +1105,7 @@ template <typename RecipeTy> class VPPhiAccessors {
return getAsRecipe()->operands();
}
- /// Returns the \p I th incoming block.
+ /// Returns the incoming block with index \p Idx.
const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
using const_incoming_block_iterator =
@@ -1121,10 +1121,7 @@ template <typename RecipeTy> class VPPhiAccessors {
}
const_incoming_block_iterator incoming_block_end() const {
return const_incoming_block_iterator(
- detail::index_iterator(getAsRecipe()->getVPDefID() ==
- VPDef::VPWidenIntOrFpInductionSC
- ? 2
- : getAsRecipe()->getNumOperands()),
+ detail::index_iterator(getAsRecipe()->getNumOperands()),
[this](size_t Idx) { return getIncomingBlock(Idx); });
}
@@ -1133,7 +1130,7 @@ template <typename RecipeTy> class VPPhiAccessors {
return make_range(incoming_block_begin(), incoming_block_end());
}
- /// Returns an iterator range over pairs of incoming values and corrsponding
+ /// Returns an iterator range over pairs of incoming values and correspondingx
/// incoming blocks.
detail::zippy<llvm::detail::zip_shortest, VPUser::const_operand_range,
const_incoming_blocks_range>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8f2142b6c3260..73f0c329e5b66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1042,24 +1042,27 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent,
}
#endif
+/// Returns the incoming block at index \p Idx for \p R. This handles both
+/// recipes placed in entry blocks of loop regions (incoming blocks are the
+/// region's predecessor and the region's exit) and other locations (incoming
+/// blocks are the direct predecessors).
static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R,
- unsigned I) {
+ unsigned Idx) {
const VPBasicBlock *Parent = R->getParent();
const VPBlockBase *Pred = nullptr;
if (Parent->getNumPredecessors() > 0) {
- Pred = Parent->getPredecessors()[I];
+ Pred = Parent->getPredecessors()[Idx];
} else {
auto *Region = Parent->getParent();
assert(Region && !Region->isReplicator() && Region->getEntry() == Parent &&
"must be in the entry block of a non-replicate region");
assert(
- I < 2 &&
- (R->getNumOperands() == 2 || isa<VPWidenIntOrFpInductionRecipe>(R)) &&
+ Idx < 2 && R->getNumOperands() == 2 &&
"when placed in an entry block, only 2 incoming blocks are available");
- // I == 0 selects the predecessor of the region, I == 1 selects the region
- // itself whose exiting block feeds the phi across the backedge.
- Pred = I == 0 ? Region->getSinglePredecessor() : Region;
+ // Idx == 0 selects the predecessor of the region, Idx == 1 selects the
+ // region itself whose exiting block feeds the phi across the backedge.
+ Pred = Idx == 0 ? Region->getSinglePredecessor() : Region;
}
return Pred->getExitingBasicBlock();
>From f768b97a4b0945c07826f5fef66adbbb85c65f6a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 26 Apr 2025 21:45:06 +0100
Subject: [PATCH 3/7] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 1 -
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6ade01cf7f7e5..e58ba815a4379 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2890,7 +2890,6 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
// Make sure the builder has a valid insert point.
Builder.SetInsertPoint(NewPhi);
-
for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c986b5f845f4e..ca7df407c3c3b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1215,7 +1215,7 @@ template <typename RecipeTy> class VPPhiAccessors {
return make_range(incoming_block_begin(), incoming_block_end());
}
- /// Returns an iterator range over pairs of incoming values and correspondingx
+ /// Returns an iterator range over pairs of incoming values and corresponding
/// incoming blocks.
detail::zippy<llvm::detail::zip_shortest, VPUser::const_operand_range,
const_incoming_blocks_range>
@@ -2028,6 +2028,7 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
/// the second from the exiting block of the region.
class VPWidenPHIRecipe : public VPSingleDefRecipe,
public VPPhiAccessors<VPWidenPHIRecipe> {
+ /// Name to use for the generated IR instruction for the widened phi.
std::string Name;
public:
>From 595b05720f812c9e3f5df43f83e923059d0ea432 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 1 May 2025 22:03:25 +0100
Subject: [PATCH 4/7] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 5 ++-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 20 +++++++++
llvm/lib/Transforms/Vectorize/VPlan.h | 43 +++++--------------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 32 --------------
4 files changed, 34 insertions(+), 66 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6c1c7126269b0..d2bbc10642114 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2883,8 +2883,11 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
// Make sure the builder has a valid insert point.
Builder.SetInsertPoint(NewPhi);
- for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
+ for (unsigned Idx = 0; Idx < VPPhi->getNumIncomingValues(); ++Idx) {
+ VPValue *Inc = VPPhi->getIncomingValue(Idx);
+ const VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
+ }
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 701e4bfe0623e..d5fa50ed1a839 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -790,6 +790,26 @@ InstructionCost VPBasicBlock::cost(ElementCount VF, VPCostContext &Ctx) {
return Cost;
}
+const VPBasicBlock *VPBasicBlock::getCFGPredecessor(unsigned Idx) const {
+ const VPBlockBase *Pred = nullptr;
+ if (getNumPredecessors() > 0) {
+ Pred = getPredecessors()[Idx];
+ } else {
+ auto *Region = getParent();
+ assert(Region && !Region->isReplicator() && Region->getEntry() == this &&
+ "must be in the entry block of a non-replicate region");
+ assert(
+ Idx < 2 && Region->getNumPredecessors() == 1 &&
+ "when placed in an entry block, only 2 incoming blocks are available");
+
+ // Idx == 0 selects the predecessor of the region, Idx == 1 selects the
+ // region itself whose exiting block feeds the phi across the backedge.
+ Pred = Idx == 0 ? Region->getSinglePredecessor() : Region;
+ }
+
+ return Pred->getExitingBasicBlock();
+}
+
InstructionCost VPRegionBlock::cost(ElementCount VF, VPCostContext &Ctx) {
if (!isReplicator()) {
InstructionCost Cost = 0;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 83632a1c03b18..7ae52b8285867 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1180,42 +1180,13 @@ template <typename RecipeTy> class VPPhiAccessors {
return getAsRecipe()->getOperand(Idx);
}
- /// Returns an interator range over the incoming values
- VPUser::const_operand_range incoming_values() const {
- return getAsRecipe()->operands();
- }
-
/// Returns the incoming block with index \p Idx.
- const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
-
- using const_incoming_block_iterator =
- mapped_iterator<detail::index_iterator,
- std::function<const VPBasicBlock *(size_t)>>;
- using const_incoming_blocks_range =
- iterator_range<const_incoming_block_iterator>;
-
- const_incoming_block_iterator incoming_block_begin() const {
- return const_incoming_block_iterator(
- detail::index_iterator(0),
- [this](size_t Idx) { return getIncomingBlock(Idx); });
- }
- const_incoming_block_iterator incoming_block_end() const {
- return const_incoming_block_iterator(
- detail::index_iterator(getAsRecipe()->getNumOperands()),
- [this](size_t Idx) { return getIncomingBlock(Idx); });
+ const VPBasicBlock *getIncomingBlock(unsigned Idx) const {
+ return getAsRecipe()->getParent()->getCFGPredecessor(Idx);
}
- /// Returns an iterator range over the incoming blocks.
- const_incoming_blocks_range incoming_blocks() const {
- return make_range(incoming_block_begin(), incoming_block_end());
- }
-
- /// Returns an iterator range over pairs of incoming values and corresponding
- /// incoming blocks.
- detail::zippy<llvm::detail::zip_shortest, VPUser::const_operand_range,
- const_incoming_blocks_range>
- incoming_values_and_blocks() const {
- return zip(incoming_values(), incoming_blocks());
+ unsigned getNumIncomingValues() const {
+ return getAsRecipe()->getNumOperands();
}
};
@@ -3372,6 +3343,12 @@ class VPBasicBlock : public VPBlockBase {
/// the cloned recipes.
VPBasicBlock *clone() override;
+ /// Returns the predecessor block at index \p Idx with the predecessors as per
+ /// the corresponding plain CFG. If the block is an entry block to a region,
+ /// the first predecessor is the single predecessor of a region, and the
+ /// second predecessor is the exiting block of the region.
+ const VPBasicBlock *getCFGPredecessor(unsigned Idx) const;
+
protected:
/// Execute the recipes in the IR basic block \p BB.
void executeRecipes(VPTransformState *State, BasicBlock *BB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b853babd61cf7..265aa7a3ae4df 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1205,32 +1205,6 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
}
#endif
-/// Returns the incoming block at index \p Idx for \p R. This handles both
-/// recipes placed in entry blocks of loop regions (incoming blocks are the
-/// region's predecessor and the region's exit) and other locations (incoming
-/// blocks are the direct predecessors).
-static const VPBasicBlock *getIncomingBlockForRecipe(const VPRecipeBase *R,
- unsigned Idx) {
- const VPBasicBlock *Parent = R->getParent();
- const VPBlockBase *Pred = nullptr;
- if (Parent->getNumPredecessors() > 0) {
- Pred = Parent->getPredecessors()[Idx];
- } else {
- auto *Region = Parent->getParent();
- assert(Region && !Region->isReplicator() && Region->getEntry() == Parent &&
- "must be in the entry block of a non-replicate region");
- assert(
- Idx < 2 && R->getNumOperands() == 2 &&
- "when placed in an entry block, only 2 incoming blocks are available");
-
- // Idx == 0 selects the predecessor of the region, Idx == 1 selects the
- // region itself whose exiting block feeds the phi across the backedge.
- Pred = Idx == 0 ? Region->getSinglePredecessor() : Region;
- }
-
- return Pred->getExitingBasicBlock();
-}
-
void VPIRMetadata::applyMetadata(Instruction &I) const {
for (const auto &[Kind, Node] : Metadata)
I.setMetadata(Kind, Node);
@@ -3712,12 +3686,6 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
-template <>
-const VPBasicBlock *
-VPPhiAccessors<VPWidenPHIRecipe>::getIncomingBlock(unsigned Idx) const {
- return getIncomingBlockForRecipe(getAsRecipe(), Idx);
-}
-
void VPWidenPHIRecipe::execute(VPTransformState &State) {
assert(EnableVPlanNativePath &&
"Non-native vplans are not expected to have VPWidenPHIRecipes.");
>From 46780ab5e27274843eb0bdb6bfc04baa06d28d93 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 3 May 2025 10:52:21 +0100
Subject: [PATCH 5/7] !fixup address comments, make getAsRecipe pure virtual.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 7 +++----
llvm/lib/Transforms/Vectorize/VPlan.h | 26 +++++++++++++++----------
2 files changed, 19 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index d5fa50ed1a839..ff7fabaaa4d7a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -798,15 +798,14 @@ const VPBasicBlock *VPBasicBlock::getCFGPredecessor(unsigned Idx) const {
auto *Region = getParent();
assert(Region && !Region->isReplicator() && Region->getEntry() == this &&
"must be in the entry block of a non-replicate region");
- assert(
- Idx < 2 && Region->getNumPredecessors() == 1 &&
- "when placed in an entry block, only 2 incoming blocks are available");
+ assert(Idx < 2 && Region->getNumPredecessors() == 1 &&
+ "loop region has a single predecessor (preheader), its entry block "
+ "has 2 incoming blocks");
// Idx == 0 selects the predecessor of the region, Idx == 1 selects the
// region itself whose exiting block feeds the phi across the backedge.
Pred = Idx == 0 ? Region->getSinglePredecessor() : Region;
}
-
return Pred->getExitingBasicBlock();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 7ae52b8285867..2768c32d24164 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1167,23 +1167,22 @@ class VPIRInstruction : public VPRecipeBase {
};
/// Helper type to provide functions to access incoming values and blocks for
-/// phi-like recipes. RecipeTy must be a sub-class of VPRecipeBase.
-template <typename RecipeTy> class VPPhiAccessors {
+/// phi-like recipes.
+class VPPhiAccessors {
+protected:
/// Return a VPRecipeBase* to the current object.
- const VPRecipeBase *getAsRecipe() const {
- return static_cast<const RecipeTy *>(this);
- }
+ virtual const VPRecipeBase *getAsRecipe() const = 0;
public:
+ virtual ~VPPhiAccessors() = default;
+
/// Returns the incoming VPValue with index \p Idx.
VPValue *getIncomingValue(unsigned Idx) const {
return getAsRecipe()->getOperand(Idx);
}
/// Returns the incoming block with index \p Idx.
- const VPBasicBlock *getIncomingBlock(unsigned Idx) const {
- return getAsRecipe()->getParent()->getCFGPredecessor(Idx);
- }
+ const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
unsigned getNumIncomingValues() const {
return getAsRecipe()->getNumOperands();
@@ -1993,11 +1992,13 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
/// recipe is placed in an entry block to a (non-replicate) region, it must have
/// exactly 2 incoming values, the first from the predecessor of the region and
/// the second from the exiting block of the region.
-class VPWidenPHIRecipe : public VPSingleDefRecipe,
- public VPPhiAccessors<VPWidenPHIRecipe> {
+class VPWidenPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors {
/// Name to use for the generated IR instruction for the widened phi.
std::string Name;
+protected:
+ const VPRecipeBase *getAsRecipe() const override { return this; }
+
public:
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and
/// debug location \p DL.
@@ -3363,6 +3364,11 @@ class VPBasicBlock : public VPBlockBase {
BasicBlock *createEmptyBasicBlock(VPTransformState &State);
};
+inline const VPBasicBlock *
+VPPhiAccessors::getIncomingBlock(unsigned Idx) const {
+ return getAsRecipe()->getParent()->getCFGPredecessor(Idx);
+}
+
/// A special type of VPBasicBlock that wraps an existing IR basic block.
/// Recipes of the block get added before the first non-phi instruction in the
/// wrapped block.
>From 5fee077e2cc337c8db74b234b8e60fcc00daf6c4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 2 May 2025 15:32:30 +0100
Subject: [PATCH 6/7] Use virtual
[VPlan] Use VPPhiAccessors for VPIRPhi.
astOf
---
llvm/lib/Transforms/Vectorize/VPlan.h | 54 +++++++++++++++++--
.../Transforms/Vectorize/VPlanVerifier.cpp | 37 ++++++++-----
2 files changed, 75 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 2768c32d24164..84ef0ba986461 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1185,7 +1185,10 @@ class VPPhiAccessors {
const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
unsigned getNumIncomingValues() const {
- return getAsRecipe()->getNumOperands();
+ auto *R = getAsRecipe();
+ return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC
+ ? 1
+ : R->getNumOperands();
}
};
@@ -1193,7 +1196,7 @@ class VPPhiAccessors {
/// cast/dyn_cast/isa and execute() implementation. A single VPValue operand is
/// allowed, and it is used to add a new incoming value for the single
/// predecessor VPBB.
-struct VPIRPhi : public VPIRInstruction {
+struct VPIRPhi : public VPIRInstruction, public VPPhiAccessors {
VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {}
static inline bool classof(const VPRecipeBase *U) {
@@ -1210,6 +1213,9 @@ struct VPIRPhi : public VPIRInstruction {
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
+
+protected:
+ const VPRecipeBase *getAsRecipe() const override { return this; }
};
/// Helper to manage IR metadata for recipes. It filters out metadata that
@@ -1735,13 +1741,15 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags,
/// * VPWidenPointerInductionRecipe: Generate vector and scalar values for a
/// pointer induction. Produces either a vector PHI per-part or scalar values
/// per-lane based on the canonical induction.
-class VPHeaderPHIRecipe : public VPSingleDefRecipe {
+class VPHeaderPHIRecipe : public VPSingleDefRecipe, public VPPhiAccessors {
protected:
VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr,
VPValue *Start, DebugLoc DL = {})
: VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>({Start}), UnderlyingInstr, DL) {
}
+ const VPRecipeBase *getAsRecipe() const override { return this; }
+
public:
~VPHeaderPHIRecipe() override = default;
@@ -3218,6 +3226,46 @@ class VPScalarIVStepsRecipe : public VPRecipeWithIRFlags,
}
};
+/// Casting from VPRecipeBase -> VPPhiAccessors is supported for all recipe
+/// types implementing VPPhiAccessors.
+template <> struct CastIsPossible<VPPhiAccessors, const VPRecipeBase *> {
+ static inline bool isPossible(const VPRecipeBase *f) {
+ return isa<VPIRPhi, VPHeaderPHIRecipe, VPWidenPHIRecipe>(f);
+ }
+};
+/// Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the
+/// recipe types implementing VPPhiAccessors.
+template <>
+struct CastInfo<VPPhiAccessors, const VPRecipeBase *>
+ : public CastIsPossible<VPPhiAccessors, const VPRecipeBase *> {
+
+ using Self = CastInfo<VPPhiAccessors, const VPRecipeBase *>;
+
+ using CastReturnType =
+ typename cast_retty<VPPhiAccessors, VPRecipeBase *>::ret_type;
+
+ static inline VPPhiAccessors *doCast(const VPRecipeBase *R) {
+ return const_cast<VPPhiAccessors *>([R]() -> const VPPhiAccessors * {
+ switch (R->getVPDefID()) {
+ case VPDef::VPIRInstructionSC:
+ return cast<VPIRPhi>(R);
+ case VPDef::VPWidenPHISC:
+ return cast<VPWidenPHIRecipe>(R);
+ default:
+ return cast<VPHeaderPHIRecipe>(R);
+ }
+ }());
+ }
+
+ static inline VPPhiAccessors *castFailed() { return nullptr; }
+
+ static inline VPPhiAccessors *doCastIfPossible(const VPRecipeBase *f) {
+ if (!Self::isPossible(f))
+ return castFailed();
+ return doCast(f);
+ }
+};
+
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 3f10b1756d7a1..04f30003f372b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -179,8 +179,7 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!verifyPhiRecipes(VPBB))
return false;
- // Verify that defs in VPBB dominate all their uses. The current
- // implementation is still incomplete.
+ // Verify that defs in VPBB dominate all their uses.
DenseMap<const VPRecipeBase *, unsigned> RecipeNumbering;
unsigned Cnt = 0;
for (const VPRecipeBase &R : *VPBB)
@@ -207,25 +206,37 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
for (const VPUser *U : V->users()) {
auto *UI = cast<VPRecipeBase>(U);
- // TODO: check dominance of incoming values for phis properly.
- if (!UI ||
- isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe,
- VPIRPhi>(UI) ||
- (isa<VPInstruction>(UI) &&
- cast<VPInstruction>(UI)->getOpcode() == Instruction::PHI))
+ const VPBlockBase *UserVPBB = UI->getParent();
+ if (auto *Phi = dyn_cast<VPPhiAccessors>(UI)) {
+ for (unsigned Idx = 0; Idx != Phi->getNumIncomingValues(); ++Idx) {
+ VPValue *IncVPV = Phi->getIncomingValue(Idx);
+ const VPBasicBlock *IncVPBB = Phi->getIncomingBlock(Idx);
+ if (IncVPV != V)
+ continue;
+ if (IncVPBB != VPBB && !VPDT.dominates(VPBB, IncVPBB)) {
+ errs() << "Use before def!\n";
+ return false;
+ }
+ }
+ continue;
+ }
+ if (isa<VPPredInstPHIRecipe>(UI))
continue;
- // If the user is in the same block, check it comes after R in the
- // block.
- if (UI->getParent() == VPBB) {
+ if (auto *VPI = dyn_cast<VPInstruction>(UI)) {
+ if (VPI->getOpcode() == Instruction::PHI)
+ continue;
+ }
+ // If the user is in the same block, check it comes after R in
+ // the block.
+ if (UserVPBB == VPBB) {
if (RecipeNumbering[UI] < RecipeNumbering[&R]) {
errs() << "Use before def!\n";
return false;
}
- continue;
}
- if (!VPDT.dominates(VPBB, UI->getParent())) {
+ if (!VPDT.dominates(VPBB, UserVPBB)) {
errs() << "Use before def!\n";
return false;
}
>From 60d5ec95f83354a57578f443bdaa4b02433722f5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 11 May 2025 19:11:17 +0100
Subject: [PATCH 7/7] !fixup update after merging f2e62cfca5e571.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 9 ++++++++-
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 4 ----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0d7890e81e4f8..4068d5b7c1119 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1135,7 +1135,14 @@ class VPPhiAccessors {
const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
/// Returns the number of incoming values, also number of incoming blocks.
- unsigned getNumIncoming() const { return getAsRecipe()->getNumOperands(); }
+ /// Note that at the moment, VPWidenIntOrFpInductionRecipes only have a single
+ /// incoming value, its start value.
+ unsigned getNumIncoming() const {
+ const VPRecipeBase *R = getAsRecipe();
+ return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC
+ ? 1
+ : getAsRecipe()->getNumOperands();
+ }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index a955b165e8133..e36f12ae47c09 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -235,10 +235,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (isa<VPPredInstPHIRecipe>(UI))
continue;
- if (auto *VPI = dyn_cast<VPInstruction>(UI)) {
- if (VPI->getOpcode() == Instruction::PHI)
- continue;
- }
// If the user is in the same block, check it comes after R in
// the block.
if (UserVPBB == VPBB) {
More information about the llvm-commits
mailing list