[llvm] [SLP][modularisation][NFC] Move order/mask helpers to SLPUtils (PR #221663)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 00:27:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Madhur Amilkanthwar (madhur13490)
<details>
<summary>Changes</summary>
Move the following BoUpSLP-independent order/mask helpers out of SLPVectorizer.cpp into SLPVectorizer/SLPUtils.{h,cpp}:
reorderReuses
reorderOrder
isReverseOrder
isRepeatedNonIdentityClusteredMask
combineOrders
They sit with the existing permutation helpers. Behavior is unchanged.
Part of the SLPVectorizer.cpp modularization effort: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
---
Full diff: https://github.com/llvm/llvm-project/pull/221663.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (-108)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp (+98)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h (+22)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a9766442856e..f074b8e0ecda2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5487,67 +5487,6 @@ BoUpSLP::~BoUpSLP() {
#endif
}
-/// Reorders the given \p Reuses mask according to the given \p Mask. \p Reuses
-/// contains original mask for the scalars reused in the node. Procedure
-/// transform this mask in accordance with the given \p Mask.
-static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
- assert(!Mask.empty() && Reuses.size() == Mask.size() &&
- "Expected non-empty mask.");
- SmallVector<int> Prev(Reuses.begin(), Reuses.end());
- Prev.swap(Reuses);
- for (unsigned I = 0, E = Prev.size(); I < E; ++I)
- if (Mask[I] != PoisonMaskElem)
- Reuses[Mask[I]] = Prev[I];
-}
-
-/// Reorders the given \p Order according to the given \p Mask. \p Order - is
-/// the original order of the scalars. Procedure transforms the provided order
-/// in accordance with the given \p Mask. If the resulting \p Order is just an
-/// identity order, \p Order is cleared.
-static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
- bool BottomOrder = false) {
- assert(!Mask.empty() && "Expected non-empty mask.");
- unsigned Sz = Mask.size();
- if (BottomOrder) {
- SmallVector<unsigned> PrevOrder;
- if (Order.empty()) {
- PrevOrder.resize(Sz);
- std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
- } else {
- PrevOrder.swap(Order);
- }
- Order.assign(Sz, Sz);
- for (unsigned I = 0; I < Sz; ++I)
- if (Mask[I] != PoisonMaskElem)
- Order[I] = PrevOrder[Mask[I]];
- if (all_of(enumerate(Order), [&](const auto &Data) {
- return Data.value() == Sz || Data.index() == Data.value();
- })) {
- Order.clear();
- return;
- }
- fixupOrderingIndices(Order);
- return;
- }
- SmallVector<int> MaskOrder;
- if (Order.empty()) {
- MaskOrder.resize(Sz);
- std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
- } else {
- inversePermutation(Order, MaskOrder);
- }
- reorderReuses(MaskOrder, Mask);
- if (ShuffleVectorInst::isIdentityMask(MaskOrder, Sz)) {
- Order.clear();
- return;
- }
- Order.assign(Sz, Sz);
- for (unsigned I = 0; I < Sz; ++I)
- if (MaskOrder[I] != PoisonMaskElem)
- Order[MaskOrder[I]] = I;
- fixupOrderingIndices(Order);
-}
-
std::optional<BoUpSLP::OrdersType>
BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
bool TopToBottom, bool IgnoreReorder) {
@@ -5769,16 +5708,6 @@ static Align computeCommonAlignment(ArrayRef<Value *> VL) {
return CommonAlignment;
}
-/// Check if \p Order represents reverse order.
-static bool isReverseOrder(ArrayRef<unsigned> Order) {
- assert(!Order.empty() &&
- "Order is empty. Please check it before using isReverseOrder.");
- unsigned Sz = Order.size();
- return all_of(enumerate(Order), [&](const auto &Pair) {
- return Pair.value() == Sz || Sz - Pair.index() - 1 == Pair.value();
- });
-}
-
/// Checks if the provided list of pointers \p Pointers represents the strided
/// pointers for type ElemTy. If they are not, nullptr is returned.
/// Otherwise, SCEV* of the stride value is returned.
@@ -7440,21 +7369,6 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
return std::nullopt;
}
-/// Checks if the given mask is a "clustered" mask with the same clusters of
-/// size \p Sz, which are not identity submasks.
-static bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask,
- unsigned Sz) {
- ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
- if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
- return false;
- for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
- ArrayRef<int> Cluster = Mask.slice(I, Sz);
- if (Cluster != FirstCluster)
- return false;
- }
- return true;
-}
-
void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
// Reorder reuses mask.
reorderReuses(TE.ReuseShuffleIndices, Mask);
@@ -7482,28 +7396,6 @@ void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
std::iota(It, std::next(It, Sz), 0);
}
-static void combineOrders(MutableArrayRef<unsigned> Order,
- ArrayRef<unsigned> SecondaryOrder) {
- assert((SecondaryOrder.empty() || Order.size() == SecondaryOrder.size()) &&
- "Expected same size of orders");
- size_t Sz = Order.size();
- SmallBitVector UsedIndices(Sz);
- for (unsigned Idx : seq<unsigned>(0, Sz)) {
- if (Order[Idx] != Sz)
- UsedIndices.set(Order[Idx]);
- }
- if (SecondaryOrder.empty()) {
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (Order[Idx] == Sz && !UsedIndices.test(Idx))
- Order[Idx] = Idx;
- } else {
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (SecondaryOrder[Idx] != Sz && Order[Idx] == Sz &&
- !UsedIndices.test(SecondaryOrder[Idx]))
- Order[Idx] = SecondaryOrder[Idx];
- }
-}
-
bool BoUpSLP::isProfitableToReorder() const {
if (DisableTreeReorder)
return false;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 94b16e9186523..d57e5f1af911b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -25,6 +25,7 @@
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
+#include <numeric>
#include <string>
#include <type_traits>
@@ -314,6 +315,103 @@ void reorderScalars(SmallVectorImpl<Value *> &Scalars, ArrayRef<int> Mask) {
Scalars[Mask[I]] = Prev[I];
}
+void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
+ assert(!Mask.empty() && Reuses.size() == Mask.size() &&
+ "Expected non-empty mask.");
+ SmallVector<int> Prev(Reuses.begin(), Reuses.end());
+ Prev.swap(Reuses);
+ for (unsigned I = 0, E = Prev.size(); I < E; ++I)
+ if (Mask[I] != PoisonMaskElem)
+ Reuses[Mask[I]] = Prev[I];
+}
+
+void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
+ bool BottomOrder) {
+ assert(!Mask.empty() && "Expected non-empty mask.");
+ unsigned Sz = Mask.size();
+ if (BottomOrder) {
+ SmallVector<unsigned> PrevOrder;
+ if (Order.empty()) {
+ PrevOrder.resize(Sz);
+ std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
+ } else {
+ PrevOrder.swap(Order);
+ }
+ Order.assign(Sz, Sz);
+ for (unsigned I = 0; I < Sz; ++I)
+ if (Mask[I] != PoisonMaskElem)
+ Order[I] = PrevOrder[Mask[I]];
+ if (all_of(enumerate(Order), [&](const auto &Data) {
+ return Data.value() == Sz || Data.index() == Data.value();
+ })) {
+ Order.clear();
+ return;
+ }
+ fixupOrderingIndices(Order);
+ return;
+ }
+ SmallVector<int> MaskOrder;
+ if (Order.empty()) {
+ MaskOrder.resize(Sz);
+ std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
+ } else {
+ inversePermutation(Order, MaskOrder);
+ }
+ reorderReuses(MaskOrder, Mask);
+ if (ShuffleVectorInst::isIdentityMask(MaskOrder, Sz)) {
+ Order.clear();
+ return;
+ }
+ Order.assign(Sz, Sz);
+ for (unsigned I = 0; I < Sz; ++I)
+ if (MaskOrder[I] != PoisonMaskElem)
+ Order[MaskOrder[I]] = I;
+ fixupOrderingIndices(Order);
+}
+
+bool isReverseOrder(ArrayRef<unsigned> Order) {
+ assert(!Order.empty() &&
+ "Order is empty. Please check it before using isReverseOrder.");
+ unsigned Sz = Order.size();
+ return all_of(enumerate(Order), [&](const auto &Pair) {
+ return Pair.value() == Sz || Sz - Pair.index() - 1 == Pair.value();
+ });
+}
+
+bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask, unsigned Sz) {
+ ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
+ if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
+ return false;
+ for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
+ ArrayRef<int> Cluster = Mask.slice(I, Sz);
+ if (Cluster != FirstCluster)
+ return false;
+ }
+ return true;
+}
+
+void combineOrders(MutableArrayRef<unsigned> Order,
+ ArrayRef<unsigned> SecondaryOrder) {
+ assert((SecondaryOrder.empty() || Order.size() == SecondaryOrder.size()) &&
+ "Expected same size of orders");
+ size_t Sz = Order.size();
+ SmallBitVector UsedIndices(Sz);
+ for (unsigned Idx : seq<unsigned>(0, Sz)) {
+ if (Order[Idx] != Sz)
+ UsedIndices.set(Order[Idx]);
+ }
+ if (SecondaryOrder.empty()) {
+ for (unsigned Idx : seq<unsigned>(0, Sz))
+ if (Order[Idx] == Sz && !UsedIndices.test(Idx))
+ Order[Idx] = Idx;
+ } else {
+ for (unsigned Idx : seq<unsigned>(0, Sz))
+ if (SecondaryOrder[Idx] != Sz && Order[Idx] == Sz &&
+ !UsedIndices.test(SecondaryOrder[Idx]))
+ Order[Idx] = SecondaryOrder[Idx];
+ }
+}
+
bool allSameType(ArrayRef<Value *> VL) {
assert(!VL.empty() && "Expected non-empty list of values.");
Type *Ty = VL.consume_front()->getType();
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 7a1a19c437d16..c519c41d051bb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -161,6 +161,28 @@ void inversePermutation(ArrayRef<unsigned> Indices, SmallVectorImpl<int> &Mask);
/// Reorders the list of scalars in accordance with the given \p Mask.
void reorderScalars(SmallVectorImpl<Value *> &Scalars, ArrayRef<int> Mask);
+/// Reorders the given \p Reuses mask according to the given \p Mask. \p Reuses
+/// contains original mask for the scalars reused in the node. Procedure
+/// transform this mask in accordance with the given \p Mask.
+void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask);
+
+/// Reorders the given \p Order according to the given \p Mask. \p Order - is
+/// the original order of the scalars. Procedure transforms the provided order
+/// in accordance with the given \p Mask. If the resulting \p Order is just an
+/// identity order, \p Order is cleared.
+void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
+ bool BottomOrder = false);
+
+/// Check if \p Order represents reverse order.
+bool isReverseOrder(ArrayRef<unsigned> Order);
+
+/// Checks if the given mask is a "clustered" mask with the same clusters of
+/// size \p Sz, which are not identity submasks.
+bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask, unsigned Sz);
+
+void combineOrders(MutableArrayRef<unsigned> Order,
+ ArrayRef<unsigned> SecondaryOrder);
+
/// \returns True iff every value in \p VL has the same Type as the first.
bool allSameType(ArrayRef<Value *> VL);
``````````
</details>
https://github.com/llvm/llvm-project/pull/221663
More information about the llvm-commits
mailing list