[llvm] [SLP][NFC] Extract VectorizableTree into own ForestTree class (PR #222805)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 15:48:28 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
@llvm/pr-subscribers-llvm-transforms
Author: Ryan Buchner (bababuck)
<details>
<summary>Changes</summary>
First step towards forest vectorization. Will eventually create a forest of `ForestTree`'s that can be costed and vectorized together.
---
Patch is 201.88 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222805.diff
1 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+798-717)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9cc4eac5f8752..e6a865e0e93d7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -368,6 +368,8 @@ struct ReductionVectorPart {
};
} // namespace
+struct SLPGraphView;
+
/// Bottom Up SLP Vectorizer.
class slpvectorizer::BoUpSLP {
class TreeEntry;
@@ -379,6 +381,14 @@ class slpvectorizer::BoUpSLP {
class ShuffleInstructionBuilder;
public:
+ /// Vectorization state that is needed on a per-tree basis.
+ struct ForestTree {
+ using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
+ VecTreeTy VectorizableTree;
+
+ ~ForestTree() = default;
+ };
+
/// If we decide to generate strided load / store, this struct contains all
/// the necessary info. It's fields are calculated by analyzeRtStrideCandidate
/// and analyzeConstantStrideCandidate. Note that Stride can be given either
@@ -437,42 +447,44 @@ class slpvectorizer::BoUpSLP {
/// Vectorize the tree that starts with the elements in \p VL.
/// Returns the vectorized root.
- Value *vectorizeTree();
+ Value *vectorizeTree(ForestTree &FT);
/// Vectorize the tree but with the list of externally used values \p
/// ExternallyUsedValues. Values in this MapVector can be replaced but the
/// generated extractvalue instructions.
Value *
- vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
+ vectorizeTree(ForestTree &FT,
+ const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot = nullptr,
ArrayRef<ReductionVectorPart> VectorValuesAndScales = {});
/// \returns the cost incurred by unwanted spills and fills, caused by
/// holding live values over call sites.
- InstructionCost getSpillCost();
+ InstructionCost getSpillCost(ForestTree &FT);
TargetTransformInfo::TargetCostKind getCostKind() const { return CostKind; }
/// Calculates the cost of the subtrees, trims non-profitable ones and returns
/// final cost.
InstructionCost
- calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals = {},
+ calculateTreeCostAndTrimNonProfitable(ForestTree &FT,
+ ArrayRef<Value *> VectorizedVals = {},
Instruction *RdxRoot = nullptr);
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
- InstructionCost getTreeCost(InstructionCost TreeCost,
+ InstructionCost getTreeCost(ForestTree &FT, InstructionCost TreeCost,
ArrayRef<Value *> VectorizedVals = {},
InstructionCost ReductionCost = TTI::TCC_Free,
Instruction *RdxRoot = nullptr);
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
- void buildTree(ArrayRef<Value *> Roots,
+ void buildTree(ForestTree &FT, ArrayRef<Value *> Roots,
const SmallDenseSet<Value *> &UserIgnoreLst);
/// Construct a vectorizable tree that starts at \p Roots.
- void buildTree(ArrayRef<Value *> Roots);
+ void buildTree(ForestTree &FT, ArrayRef<Value *> Roots);
/// Sets the narrowed reduction chain instructions, dropped together with
/// the reduction.
@@ -566,28 +578,33 @@ class slpvectorizer::BoUpSLP {
/// with runtime alias checks). When true, CFG analyses must not be preserved.
bool isCFGChanged() const { return CFGChanged; }
- TreeEntry &getRootNode() {
- assert(!VectorizableTree.empty() && "No graph to get the first node from");
- return *VectorizableTree.front();
+ static TreeEntry &getRootNode(ForestTree &FT) {
+ assert(!FT.VectorizableTree.empty() &&
+ "No graph to get the first node from");
+ return *FT.VectorizableTree.front();
}
- const TreeEntry &getRootNode() const {
- assert(!VectorizableTree.empty() && "No graph to get the first node from");
- return *VectorizableTree.front();
+ static const TreeEntry &getRootNode(const ForestTree &FT) {
+ assert(!FT.VectorizableTree.empty() &&
+ "No graph to get the first node from");
+ return *FT.VectorizableTree.front();
}
/// Returns the scalars of the root node.
- ArrayRef<Value *> getRootNodeScalars() const { return getRootNode().Scalars; }
+ ArrayRef<Value *> getRootNodeScalars(ForestTree &FT) const {
+ return getRootNode(FT).Scalars;
+ }
/// Returns the lane the given value is vectorized to in the root node.
- unsigned findRootLaneForValue(Value *V) const {
- return getRootNode().findLaneForValue(V);
+ unsigned findRootLaneForValue(ForestTree &FT, Value *V) const {
+ return getRootNode(FT).findLaneForValue(V);
}
/// Returns the type/is-signed info for the root node in the graph without
/// casting.
- std::optional<std::pair<Type *, bool>> getRootNodeTypeWithNoCast() const {
- const TreeEntry &Root = getRootNode();
+ std::optional<std::pair<Type *, bool>>
+ getRootNodeTypeWithNoCast(ForestTree &FT) const {
+ const TreeEntry &Root = getRootNode(FT);
if (Root.State != TreeEntry::Vectorize || Root.isAltShuffle() ||
!Root.Scalars.front()->getType()->isIntegerTy())
return std::nullopt;
@@ -605,40 +622,41 @@ class slpvectorizer::BoUpSLP {
/// Checks if the root graph node can be emitted with narrower bitwidth at
/// codegen and returns it signedness, if so.
- bool isSignedMinBitwidthRootNode() const {
- return MinBWs.at(&getRootNode()).second;
+ bool isSignedMinBitwidthRootNode(ForestTree &FT) const {
+ return MinBWs.at(&getRootNode(FT)).second;
}
/// Returns reduction type after minbitdth analysis.
- FixedVectorType *getReductionType() const {
+ FixedVectorType *getReductionType(ForestTree &FT) const {
if (ReductionBitWidth == 0 ||
- !getRootNodeScalars().front()->getType()->isIntegerTy() ||
+ !getRootNodeScalars(FT).front()->getType()->isIntegerTy() ||
ReductionBitWidth >=
- DL->getTypeSizeInBits(getRootNodeScalars().front()->getType()))
+ DL->getTypeSizeInBits(getRootNodeScalars(FT).front()->getType()))
return cast<FixedVectorType>(
- getWidenedType(getRootNodeScalars().front()->getType(),
- getRootNode().getVectorFactor()));
+ getWidenedType(getRootNodeScalars(FT).front()->getType(),
+ getRootNode(FT).getVectorFactor()));
return cast<FixedVectorType>(getWidenedType(
- IntegerType::get(getRootNodeScalars().front()->getContext(),
+ IntegerType::get(getRootNodeScalars(FT).front()->getContext(),
ReductionBitWidth),
- getRootNode().getVectorFactor()));
+ getRootNode(FT).getVectorFactor()));
}
/// Returns true if the tree results in one of the reduced bitcasts variants.
- bool isReducedBitcastRoot() const {
- return getRootNode().hasState() &&
- (getRootNode().CombinedOp == TreeEntry::ReducedBitcast ||
- getRootNode().CombinedOp == TreeEntry::ReducedBitcastBSwap ||
- getRootNode().CombinedOp == TreeEntry::ReducedBitcastLoads ||
- getRootNode().CombinedOp == TreeEntry::ReducedBitcastBSwapLoads) &&
- getRootNode().State == TreeEntry::Vectorize;
+ bool isReducedBitcastRoot(ForestTree &FT) const {
+ return getRootNode(FT).hasState() &&
+ (getRootNode(FT).CombinedOp == TreeEntry::ReducedBitcast ||
+ getRootNode(FT).CombinedOp == TreeEntry::ReducedBitcastBSwap ||
+ getRootNode(FT).CombinedOp == TreeEntry::ReducedBitcastLoads ||
+ getRootNode(FT).CombinedOp ==
+ TreeEntry::ReducedBitcastBSwapLoads) &&
+ getRootNode(FT).State == TreeEntry::Vectorize;
}
/// Returns true if the tree results in the reduced cmp bitcast root.
- bool isReducedCmpBitcastRoot() const {
- return getRootNode().hasState() &&
- getRootNode().CombinedOp == TreeEntry::ReducedCmpBitcast &&
- getRootNode().State == TreeEntry::Vectorize;
+ bool isReducedCmpBitcastRoot(ForestTree &FT) const {
+ return getRootNode(FT).hasState() &&
+ getRootNode(FT).CombinedOp == TreeEntry::ReducedCmpBitcast &&
+ getRootNode(FT).State == TreeEntry::Vectorize;
}
/// Returns true if the tree is a reduction tree.
@@ -649,14 +667,14 @@ class slpvectorizer::BoUpSLP {
/// ExternallyUsedValues contains additional list of external uses to handle
/// vectorization of reductions.
void
- buildExternalUses(const ExtraValueToDebugLocsMap &ExternallyUsedValues = {});
+ buildExternalUses(ForestTree &FT,
+ const ExtraValueToDebugLocsMap &ExternallyUsedValues = {});
/// Transforms graph nodes to target specific representations, if profitable.
- void transformNodes();
+ void transformNodes(ForestTree &FT);
/// Clear the internal data structures that are created by 'buildTree'.
void deleteTree() {
- VectorizableTree.clear();
ScalarToTreeEntries.clear();
DeletedNodes.clear();
TransformedToGatherNodes.clear();
@@ -701,7 +719,9 @@ class slpvectorizer::BoUpSLP {
MergedLoopBTCs.clear();
}
- unsigned getTreeSize() const { return VectorizableTree.size(); }
+ unsigned getTreeSize(const ForestTree &FT) const {
+ return FT.VectorizableTree.size();
+ }
/// Returns the base graph size, before any transformations.
unsigned getCanonicalGraphSize() const { return BaseGraphSize; }
@@ -733,7 +753,8 @@ class slpvectorizer::BoUpSLP {
/// \param TopToBottom If true, used for the whole tree rotation, false - for
/// sub-tree rotations. \param IgnoreReorder true, if the order of the root
/// node might be ignored.
- std::optional<OrdersType> findReusedOrderedScalars(const TreeEntry &TE,
+ std::optional<OrdersType> findReusedOrderedScalars(ForestTree &FT,
+ const TreeEntry &TE,
bool TopToBottom,
bool IgnoreReorder);
@@ -748,13 +769,15 @@ class slpvectorizer::BoUpSLP {
/// \param TopToBottom If true, include the order of vectorized stores and
/// insertelement nodes, otherwise skip them.
/// \param IgnoreReorder true, if the root node order can be ignored.
- std::optional<OrdersType>
- getReorderingData(const TreeEntry &TE, bool TopToBottom, bool IgnoreReorder);
+ std::optional<OrdersType> getReorderingData(ForestTree &FT,
+ const TreeEntry &TE,
+ bool TopToBottom,
+ bool IgnoreReorder);
/// Checks if it is profitable to reorder the current tree.
/// If the tree does not contain many profitable reordable nodes, better to
/// skip it to save compile time.
- bool isProfitableToReorder() const;
+ bool isProfitableToReorder(ForestTree &FT) const;
/// Reorders the current graph to the most profitable order starting from the
/// root node to the leaf nodes. The best order is chosen only from the nodes
@@ -762,7 +785,7 @@ class slpvectorizer::BoUpSLP {
/// parts of subgraph with smaller VF and they are reordered independently. We
/// can make it because we still need to extend smaller nodes to the wider VF
/// and we can merge reordering shuffles with the widening shuffles.
- void reorderTopToBottom();
+ void reorderTopToBottom(ForestTree &FT);
/// Reorders the current graph to the most profitable order starting from
/// leaves to the root. It allows to rotate small subgraphs and reduce the
@@ -771,7 +794,7 @@ class slpvectorizer::BoUpSLP {
/// operands. Plus, even the leaf nodes have different orders, it allows to
/// sink reordering in the graph closer to the root node and merge it later
/// during analysis.
- void reorderBottomToTop(bool IgnoreReorder = false);
+ void reorderBottomToTop(ForestTree &FT, bool IgnoreReorder = false);
/// Marks the schedule data of the copyable-modeled operands of \p TE for
/// dependency recalculation at the next bundle scheduling.
@@ -786,7 +809,7 @@ class slpvectorizer::BoUpSLP {
/// Compute the minimum type sizes required to represent the entries in a
/// vectorizable tree.
- void computeMinimumValueSizes();
+ void computeMinimumValueSizes(ForestTree &FT);
// \returns maximum vector register size as set by TTI or overridden by cl::opt.
unsigned getMaxVecRegSize() const {
@@ -841,15 +864,16 @@ class slpvectorizer::BoUpSLP {
/// inserted index for insertvalue.
FixedVectorType *getInsertBuildVectorSrcTy(const TreeEntry *E) const;
- /// \returns True if the VectorizableTree is both tiny and not fully
+ /// \returns True if the vectorizable tree is both tiny and not fully
/// vectorizable. We do not vectorize such trees.
- bool isTreeTinyAndNotFullyVectorizable(bool ForReduction = false) const;
+ bool isTreeTinyAndNotFullyVectorizable(ForestTree &FT,
+ bool ForReduction = false) const;
/// Checks if the graph and all its subgraphs cannot be better vectorized.
/// It may happen, if all gather nodes are loads and they cannot be
/// "clusterized". In this case even subgraphs cannot be vectorized more
/// effectively than the base graph.
- bool isTreeNotExtendable() const;
+ bool isTreeNotExtendable(ForestTree &FT) const;
bool isStridedLoad(ArrayRef<Value *> PointerOps, Type *ScalarTy,
Align Alignment, const int64_t Diff,
@@ -2392,7 +2416,7 @@ class slpvectorizer::BoUpSLP {
/// \param Op2 second homogeneous scalars.
/// \param ReorderIndices indices to reorder the scalars.
/// \returns true if the node was successfully built.
- bool canBuildSplitNode(ArrayRef<Value *> VL,
+ bool canBuildSplitNode(ForestTree *FT, ArrayRef<Value *> VL,
const InstructionsState &LocalState,
SmallVectorImpl<Value *> &Op1,
SmallVectorImpl<Value *> &Op2,
@@ -2405,11 +2429,14 @@ class slpvectorizer::BoUpSLP {
/// truncation. We collect the entries that will be demoted in ToDemote.
/// \param E Node for analysis
/// \param ToDemote indices of the nodes to be demoted.
- bool collectValuesToDemote(
- const TreeEntry &E, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
- SmallVectorImpl<unsigned> &ToDemote, DenseSet<const TreeEntry *> &Visited,
- const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
- bool &IsProfitableToDemote, bool IsTruncRoot) const;
+ bool collectValuesToDemote(ForestTree &FT, const TreeEntry &E,
+ bool IsProfitableToDemoteRoot, unsigned &BitWidth,
+ SmallVectorImpl<unsigned> &ToDemote,
+ DenseSet<const TreeEntry *> &Visited,
+ const SmallDenseSet<unsigned, 8> &NodesToKeepBWs,
+ unsigned &MaxDepthLevel,
+ bool &IsProfitableToDemote,
+ bool IsTruncRoot) const;
/// Builds the list of reorderable operands on the edges \p Edges of the \p
/// UserTE, which allow reordering (i.e. the operands can be reordered because
@@ -2436,12 +2463,12 @@ class slpvectorizer::BoUpSLP {
/// Estimates the number of scalar instructions in the tree, each weighted by
/// its loop-nest trip count (nest-invariant entries are dropped when
/// \p TreeLoop is non-null).
- uint64_t getNumScalarInsts(bool HasTreeLoop);
+ uint64_t getNumScalarInsts(ForestTree &FT, bool HasTreeLoop);
/// Estimates the number of vector instructions (including buildvectors,
/// shuffles, and extracts) the tree produces, weighted like
/// getNumScalarInsts().
- uint64_t getNumVectorInsts(bool HasTreeLoop);
+ uint64_t getNumVectorInsts(ForestTree &FT, bool HasTreeLoop);
/// Return information about the vector formed for the specified index
/// of a vector of (the same) instruction.
@@ -2499,7 +2526,7 @@ class slpvectorizer::BoUpSLP {
ArrayRef<const Loop *> getLoopNest(const Loop *L);
/// \returns the cost of the vectorizable entry.
- InstructionCost getEntryCost(const TreeEntry *E,
+ InstructionCost getEntryCost(ForestTree &FT, const TreeEntry *E,
ArrayRef<Value *> VectorizedVals,
SmallPtrSetImpl<Value *> &CheckedExtracts);
@@ -2508,13 +2535,13 @@ class slpvectorizer::BoUpSLP {
/// scalar/slot type used to widen into \p VecTy/\p FinalVecTy and may itself
/// be a FixedVectorType in ReVec mode or an adjusted type due to MinBWs.
InstructionCost
- getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy, Type *VecTy,
- Type *FinalVecTy,
+ getVectorSpillReloadCost(ForestTree &FT, const TreeEntry *E, Type *ScalarTy,
+ Type *VecTy, Type *FinalVecTy,
const TTI::TargetCostKind CostKind) const;
/// This is the recursive part of buildTree.
- void buildTreeRec(ArrayRef<Value *> Roots, unsigned Depth, const EdgeInfo &EI,
- unsigned InterleaveFactor = 0);
+ void buildTreeRec(ForestTree &FT, ArrayRef<Value *> Roots, unsigned Depth,
+ const EdgeInfo &EI, unsigned InterleaveFactor = 0);
/// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
/// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -2528,22 +2555,23 @@ class slpvectorizer::BoUpSLP {
bool ResizeAllowed = false) const;
/// Vectorize a single entry in the tree.
- Value *vectorizeTree(TreeEntry *E);
+ Value *vectorizeTree(ForestTree &FT, TreeEntry *E);
/// Vectorize a single entry in the tree, the \p Idx-th operand of the entry
/// \p E.
- Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx);
+ Value *vectorizeOperand(ForestTree &FT, TreeEntry *E, unsigned NodeIdx);
/// Create a new vector from a list of scalar values. Produces a sequence
/// which exploits values reused across lanes, and arranges the inserts
/// for ease of later optimization.
template <typename BVTy, typename ResTy, typename... Args>
- ResTy processBuildVector(const TreeEntry *E, Type *ScalarTy, Args &...Params);
+ ResTy processBuildVector(ForestTree &FT, const TreeEntry *E, Type *ScalarTy,
+ Args &...Params);
/// Create a new vector from a list of scalar values. Produces a sequence
/// which exploits values reused across lanes, and arranges the inserts
/// for ease of later optimization.
- Value *createBuildVector(const TreeEntry *E, Type *ScalarTy);
+ Value *createBuildVector(ForestTree &FT, const TreeEntry *E, Type *ScalarTy);
/// Returns the instruction in the bundle, which can be used as a base point
/// for scheduling. Usually it is the last instruction in the bundle, except
@@ -2582,9 +2610,9 @@ class slpvectorizer::BoUpSLP {
/// previous tree entries. \p Part of \p Mask is filled with the shuffle mask.
std::optional<TargetTransformInfo::ShuffleKind>
isGatherShuffledSingleRegisterEntry(
- const TreeEntry *TE, ArrayRef<Value *> VL, MutableArrayRef<int> Mask,
- SmallVectorImpl<const TreeEntry *> &Entries, unsigned Part, bool ForOrder,
- unsigned SliceSize);
+ ForestTree &FT, const TreeEntry *TE, ArrayRef<Value *> VL,
+ MutableArrayRef<int> Mask, SmallVectorImpl<const TreeEntry *> &Entries,
+ unsigned Part, bool ForOrder, unsigned SliceSize);
/// Checks if the gathered \p VL can be represented as multi-register
/// shuffle(s) of previous tree entries.
@@ -2599,7 +2627,8 @@ class slpvectorizer::BoUpSLP {
/// the shuffle mask (also on per-register base).
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>>
isGatherShuffledEntry(
- const TreeEntry *TE, ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
+ ForestTree &FT, const TreeEntry *TE, ArrayRef<Value *> VL,
+ SmallVectorImpl<int> &Mask,
SmallVectorImpl<SmallVector<const TreeEntry *>> &Entries,
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222805
More information about the llvm-commits
mailing list