[llvm] [SLP][modularisation][NFC] Move order/mask helpers to SLPUtils (PR #221663)
Madhur Amilkanthwar via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 06:50:41 PDT 2026
https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/221663
>From a2ef893859afb3a7323d8004908e4a5a92aa2a43 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Mon, 7 Sep 2026 00:18:32 -0700
Subject: [PATCH 1/2] [SLP][modularisation][NFC] Move order/mask helpers to
SLPUtils
Move the following BoUpSLP-independent order/mask helpers out of
SLPVectorizer.cpp into SLPVectorizer/SLPUtils.{h,cpp}:
reorderReuses
reorderOrder
isReverseOrder
isRepeatedNonIdentityClusteredMask
combineOrders
They sit with the existing permutation helpers. Behavior is unchanged.
Part of the SLPVectorizer.cpp modularization effort:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 108 ------------------
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 98 ++++++++++++++++
.../Vectorize/SLPVectorizer/SLPUtils.h | 22 ++++
3 files changed, 120 insertions(+), 108 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a9766442856e..f074b8e0ecda2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5487,67 +5487,6 @@ BoUpSLP::~BoUpSLP() {
#endif
}
-/// Reorders the given \p Reuses mask according to the given \p Mask. \p Reuses
-/// contains original mask for the scalars reused in the node. Procedure
-/// transform this mask in accordance with the given \p Mask.
-static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
- assert(!Mask.empty() && Reuses.size() == Mask.size() &&
- "Expected non-empty mask.");
- SmallVector<int> Prev(Reuses.begin(), Reuses.end());
- Prev.swap(Reuses);
- for (unsigned I = 0, E = Prev.size(); I < E; ++I)
- if (Mask[I] != PoisonMaskElem)
- Reuses[Mask[I]] = Prev[I];
-}
-
-/// Reorders the given \p Order according to the given \p Mask. \p Order - is
-/// the original order of the scalars. Procedure transforms the provided order
-/// in accordance with the given \p Mask. If the resulting \p Order is just an
-/// identity order, \p Order is cleared.
-static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
- bool BottomOrder = false) {
- assert(!Mask.empty() && "Expected non-empty mask.");
- unsigned Sz = Mask.size();
- if (BottomOrder) {
- SmallVector<unsigned> PrevOrder;
- if (Order.empty()) {
- PrevOrder.resize(Sz);
- std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
- } else {
- PrevOrder.swap(Order);
- }
- Order.assign(Sz, Sz);
- for (unsigned I = 0; I < Sz; ++I)
- if (Mask[I] != PoisonMaskElem)
- Order[I] = PrevOrder[Mask[I]];
- if (all_of(enumerate(Order), [&](const auto &Data) {
- return Data.value() == Sz || Data.index() == Data.value();
- })) {
- Order.clear();
- return;
- }
- fixupOrderingIndices(Order);
- return;
- }
- SmallVector<int> MaskOrder;
- if (Order.empty()) {
- MaskOrder.resize(Sz);
- std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
- } else {
- inversePermutation(Order, MaskOrder);
- }
- reorderReuses(MaskOrder, Mask);
- if (ShuffleVectorInst::isIdentityMask(MaskOrder, Sz)) {
- Order.clear();
- return;
- }
- Order.assign(Sz, Sz);
- for (unsigned I = 0; I < Sz; ++I)
- if (MaskOrder[I] != PoisonMaskElem)
- Order[MaskOrder[I]] = I;
- fixupOrderingIndices(Order);
-}
-
std::optional<BoUpSLP::OrdersType>
BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
bool TopToBottom, bool IgnoreReorder) {
@@ -5769,16 +5708,6 @@ static Align computeCommonAlignment(ArrayRef<Value *> VL) {
return CommonAlignment;
}
-/// Check if \p Order represents reverse order.
-static bool isReverseOrder(ArrayRef<unsigned> Order) {
- assert(!Order.empty() &&
- "Order is empty. Please check it before using isReverseOrder.");
- unsigned Sz = Order.size();
- return all_of(enumerate(Order), [&](const auto &Pair) {
- return Pair.value() == Sz || Sz - Pair.index() - 1 == Pair.value();
- });
-}
-
/// Checks if the provided list of pointers \p Pointers represents the strided
/// pointers for type ElemTy. If they are not, nullptr is returned.
/// Otherwise, SCEV* of the stride value is returned.
@@ -7440,21 +7369,6 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
return std::nullopt;
}
-/// Checks if the given mask is a "clustered" mask with the same clusters of
-/// size \p Sz, which are not identity submasks.
-static bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask,
- unsigned Sz) {
- ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
- if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
- return false;
- for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
- ArrayRef<int> Cluster = Mask.slice(I, Sz);
- if (Cluster != FirstCluster)
- return false;
- }
- return true;
-}
-
void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
// Reorder reuses mask.
reorderReuses(TE.ReuseShuffleIndices, Mask);
@@ -7482,28 +7396,6 @@ void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
std::iota(It, std::next(It, Sz), 0);
}
-static void combineOrders(MutableArrayRef<unsigned> Order,
- ArrayRef<unsigned> SecondaryOrder) {
- assert((SecondaryOrder.empty() || Order.size() == SecondaryOrder.size()) &&
- "Expected same size of orders");
- size_t Sz = Order.size();
- SmallBitVector UsedIndices(Sz);
- for (unsigned Idx : seq<unsigned>(0, Sz)) {
- if (Order[Idx] != Sz)
- UsedIndices.set(Order[Idx]);
- }
- if (SecondaryOrder.empty()) {
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (Order[Idx] == Sz && !UsedIndices.test(Idx))
- Order[Idx] = Idx;
- } else {
- for (unsigned Idx : seq<unsigned>(0, Sz))
- if (SecondaryOrder[Idx] != Sz && Order[Idx] == Sz &&
- !UsedIndices.test(SecondaryOrder[Idx]))
- Order[Idx] = SecondaryOrder[Idx];
- }
-}
-
bool BoUpSLP::isProfitableToReorder() const {
if (DisableTreeReorder)
return false;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 94b16e9186523..d57e5f1af911b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -25,6 +25,7 @@
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
+#include <numeric>
#include <string>
#include <type_traits>
@@ -314,6 +315,103 @@ void reorderScalars(SmallVectorImpl<Value *> &Scalars, ArrayRef<int> Mask) {
Scalars[Mask[I]] = Prev[I];
}
+void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
+ assert(!Mask.empty() && Reuses.size() == Mask.size() &&
+ "Expected non-empty mask.");
+ SmallVector<int> Prev(Reuses.begin(), Reuses.end());
+ Prev.swap(Reuses);
+ for (unsigned I = 0, E = Prev.size(); I < E; ++I)
+ if (Mask[I] != PoisonMaskElem)
+ Reuses[Mask[I]] = Prev[I];
+}
+
+void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
+ bool BottomOrder) {
+ assert(!Mask.empty() && "Expected non-empty mask.");
+ unsigned Sz = Mask.size();
+ if (BottomOrder) {
+ SmallVector<unsigned> PrevOrder;
+ if (Order.empty()) {
+ PrevOrder.resize(Sz);
+ std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
+ } else {
+ PrevOrder.swap(Order);
+ }
+ Order.assign(Sz, Sz);
+ for (unsigned I = 0; I < Sz; ++I)
+ if (Mask[I] != PoisonMaskElem)
+ Order[I] = PrevOrder[Mask[I]];
+ if (all_of(enumerate(Order), [&](const auto &Data) {
+ return Data.value() == Sz || Data.index() == Data.value();
+ })) {
+ Order.clear();
+ return;
+ }
+ fixupOrderingIndices(Order);
+ return;
+ }
+ SmallVector<int> MaskOrder;
+ if (Order.empty()) {
+ MaskOrder.resize(Sz);
+ std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
+ } else {
+ inversePermutation(Order, MaskOrder);
+ }
+ reorderReuses(MaskOrder, Mask);
+ if (ShuffleVectorInst::isIdentityMask(MaskOrder, Sz)) {
+ Order.clear();
+ return;
+ }
+ Order.assign(Sz, Sz);
+ for (unsigned I = 0; I < Sz; ++I)
+ if (MaskOrder[I] != PoisonMaskElem)
+ Order[MaskOrder[I]] = I;
+ fixupOrderingIndices(Order);
+}
+
+bool isReverseOrder(ArrayRef<unsigned> Order) {
+ assert(!Order.empty() &&
+ "Order is empty. Please check it before using isReverseOrder.");
+ unsigned Sz = Order.size();
+ return all_of(enumerate(Order), [&](const auto &Pair) {
+ return Pair.value() == Sz || Sz - Pair.index() - 1 == Pair.value();
+ });
+}
+
+bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask, unsigned Sz) {
+ ArrayRef<int> FirstCluster = Mask.slice(0, Sz);
+ if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz))
+ return false;
+ for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) {
+ ArrayRef<int> Cluster = Mask.slice(I, Sz);
+ if (Cluster != FirstCluster)
+ return false;
+ }
+ return true;
+}
+
+void combineOrders(MutableArrayRef<unsigned> Order,
+ ArrayRef<unsigned> SecondaryOrder) {
+ assert((SecondaryOrder.empty() || Order.size() == SecondaryOrder.size()) &&
+ "Expected same size of orders");
+ size_t Sz = Order.size();
+ SmallBitVector UsedIndices(Sz);
+ for (unsigned Idx : seq<unsigned>(0, Sz)) {
+ if (Order[Idx] != Sz)
+ UsedIndices.set(Order[Idx]);
+ }
+ if (SecondaryOrder.empty()) {
+ for (unsigned Idx : seq<unsigned>(0, Sz))
+ if (Order[Idx] == Sz && !UsedIndices.test(Idx))
+ Order[Idx] = Idx;
+ } else {
+ for (unsigned Idx : seq<unsigned>(0, Sz))
+ if (SecondaryOrder[Idx] != Sz && Order[Idx] == Sz &&
+ !UsedIndices.test(SecondaryOrder[Idx]))
+ Order[Idx] = SecondaryOrder[Idx];
+ }
+}
+
bool allSameType(ArrayRef<Value *> VL) {
assert(!VL.empty() && "Expected non-empty list of values.");
Type *Ty = VL.consume_front()->getType();
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 7a1a19c437d16..c519c41d051bb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -161,6 +161,28 @@ void inversePermutation(ArrayRef<unsigned> Indices, SmallVectorImpl<int> &Mask);
/// Reorders the list of scalars in accordance with the given \p Mask.
void reorderScalars(SmallVectorImpl<Value *> &Scalars, ArrayRef<int> Mask);
+/// Reorders the given \p Reuses mask according to the given \p Mask. \p Reuses
+/// contains original mask for the scalars reused in the node. Procedure
+/// transform this mask in accordance with the given \p Mask.
+void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask);
+
+/// Reorders the given \p Order according to the given \p Mask. \p Order - is
+/// the original order of the scalars. Procedure transforms the provided order
+/// in accordance with the given \p Mask. If the resulting \p Order is just an
+/// identity order, \p Order is cleared.
+void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
+ bool BottomOrder = false);
+
+/// Check if \p Order represents reverse order.
+bool isReverseOrder(ArrayRef<unsigned> Order);
+
+/// Checks if the given mask is a "clustered" mask with the same clusters of
+/// size \p Sz, which are not identity submasks.
+bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask, unsigned Sz);
+
+void combineOrders(MutableArrayRef<unsigned> Order,
+ ArrayRef<unsigned> SecondaryOrder);
+
/// \returns True iff every value in \p VL has the same Type as the first.
bool allSameType(ArrayRef<Value *> VL);
>From d673b37b4c3f5c157a0b311e3b9aefe690240f23 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Mon, 7 Sep 2026 06:48:21 -0700
Subject: [PATCH 2/2] fixup! [SLP][modularisation][NFC] Move order/mask helpers
to SLPUtils
---
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index c519c41d051bb..0fff68e052106 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -180,6 +180,10 @@ bool isReverseOrder(ArrayRef<unsigned> Order);
/// size \p Sz, which are not identity submasks.
bool isRepeatedNonIdentityClusteredMask(ArrayRef<int> Mask, unsigned Sz);
+/// Fills unset elements of \p Order (marked with the sentinel value equal to
+/// the order size) with the corresponding elements of \p SecondaryOrder,
+/// skipping already used indices, or with the identity order if
+/// \p SecondaryOrder is empty.
void combineOrders(MutableArrayRef<unsigned> Order,
ArrayRef<unsigned> SecondaryOrder);
More information about the llvm-commits
mailing list