[llvm] [SLP][modularisation][NFC] Move clusterSortPtrAccesses to SLPMemoryUtils (PR #222235)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 22:03:20 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Madhur Amilkanthwar (madhur13490)
<details>
<summary>Changes</summary>
Move the BoUpSLP-independent pointer-clustering sort out of SLPVectorizer.cpp into SLPVectorizer/SLPMemoryUtils.{h,cpp}.
clusterSortPtrAccesses reads the file-local RecursionMaxDepth cl::opt via getUnderlyingObject; the option stays static in SLPVectorizer.cpp and the moved helper takes its value as an explicit unsigned parameter (also captured by the local comparison lambda). Behavior is unchanged.
Part of the SLPVectorizer.cpp modularization effort: https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
Assisted by AI
---
Full diff: https://github.com/llvm/llvm-project/pull/222235.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+2-106)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp (+108)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h (+9)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index defcdc331203d..07aacb4d770b7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6428,111 +6428,6 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
return LoadsState::Gather;
}
-static bool clusterSortPtrAccesses(ArrayRef<Value *> VL,
- ArrayRef<BasicBlock *> BBs, Type *ElemTy,
- const DataLayout &DL, ScalarEvolution &SE,
- SmallVectorImpl<unsigned> &SortedIndices) {
- assert(
- all_of(VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
- "Expected list of pointer operands.");
- // Map from bases to a vector of (Ptr, Offset, OrigIdx), which we insert each
- // Ptr into, sort and return the sorted indices with values next to one
- // another.
- SmallMapVector<
- std::pair<BasicBlock *, Value *>,
- SmallVector<SmallVector<std::tuple<Value *, int64_t, unsigned>>>, 8>
- Bases;
- Bases
- .try_emplace(std::make_pair(
- BBs.front(), getUnderlyingObject(VL.front(), RecursionMaxDepth)))
- .first->second.emplace_back().emplace_back(VL.front(), 0U, 0U);
-
- SortedIndices.clear();
- for (auto [Cnt, Ptr] : enumerate(VL.drop_front())) {
- auto Key = std::make_pair(BBs[Cnt + 1],
- getUnderlyingObject(Ptr, RecursionMaxDepth));
- bool Found = any_of(Bases.try_emplace(Key).first->second,
- [&, &Cnt = Cnt, &Ptr = Ptr](auto &Base) {
- std::optional<int64_t> Diff =
- getPointersDiff(ElemTy, std::get<0>(Base.front()),
- ElemTy, Ptr, DL, SE,
- /*StrictCheck=*/true);
- if (!Diff)
- return false;
-
- Base.emplace_back(Ptr, *Diff, Cnt + 1);
- return true;
- });
-
- if (!Found) {
- // If we haven't found enough to usefully cluster, return early.
- if (Bases.size() > VL.size() / 2 - 1)
- return false;
-
- // Not found already - add a new Base
- Bases.find(Key)->second.emplace_back().emplace_back(Ptr, 0, Cnt + 1);
- }
- }
-
- if (Bases.size() == VL.size())
- return false;
-
- if (Bases.size() == 1 && (Bases.front().second.size() == 1 ||
- Bases.front().second.size() == VL.size()))
- return false;
-
- // For each of the bases sort the pointers by Offset and check if any of the
- // base become consecutively allocated.
- auto ComparePointers = [](Value *Ptr1, Value *Ptr2) {
- SmallPtrSet<Value *, 13> FirstPointers;
- SmallPtrSet<Value *, 13> SecondPointers;
- Value *P1 = Ptr1;
- Value *P2 = Ptr2;
- unsigned Depth = 0;
- while (!FirstPointers.contains(P2) && !SecondPointers.contains(P1)) {
- if (P1 == P2 || Depth > RecursionMaxDepth)
- return false;
- FirstPointers.insert(P1);
- SecondPointers.insert(P2);
- P1 = getUnderlyingObject(P1, /*MaxLookup=*/1);
- P2 = getUnderlyingObject(P2, /*MaxLookup=*/1);
- ++Depth;
- }
- assert((FirstPointers.contains(P2) || SecondPointers.contains(P1)) &&
- "Unable to find matching root.");
- return FirstPointers.contains(P2) && !SecondPointers.contains(P1);
- };
- for (auto &Base : Bases) {
- for (auto &Vec : Base.second) {
- if (Vec.size() > 1) {
- stable_sort(Vec, llvm::less_second());
- int64_t InitialOffset = std::get<1>(Vec[0]);
- bool AnyConsecutive =
- all_of(enumerate(Vec), [InitialOffset](const auto &P) {
- return std::get<1>(P.value()) ==
- int64_t(P.index()) + InitialOffset;
- });
- // Fill SortedIndices array only if it looks worth-while to sort the
- // ptrs.
- if (!AnyConsecutive)
- return false;
- }
- }
- stable_sort(Base.second, [&](const auto &V1, const auto &V2) {
- return ComparePointers(std::get<0>(V1.front()), std::get<0>(V2.front()));
- });
- }
-
- for (auto &T : Bases)
- for (const auto &Vec : T.second)
- for (const auto &P : Vec)
- SortedIndices.push_back(std::get<2>(P));
-
- assert(SortedIndices.size() == VL.size() &&
- "Expected SortedIndices to be the size of VL");
- return true;
-}
-
std::optional<BoUpSLP::OrdersType>
BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
assert(TE.isGather() && "Expected gather node only.");
@@ -6552,7 +6447,8 @@ BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
BoUpSLP::OrdersType Order;
if (!LoadEntriesToVectorize.contains(TE.Idx) &&
- clusterSortPtrAccesses(Ptrs, BBs, ScalarTy, *DL, *SE, Order))
+ clusterSortPtrAccesses(Ptrs, BBs, ScalarTy, *DL, *SE, RecursionMaxDepth,
+ Order))
return std::move(Order);
return std::nullopt;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
index ced6baf782628..382adb71cede2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -13,8 +13,10 @@
#include "SLPUtils.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/ScalarEvolution.h"
@@ -31,6 +33,7 @@
#include <limits>
#include <optional>
#include <set>
+#include <tuple>
#include <utility>
using namespace llvm;
@@ -409,4 +412,109 @@ bool isMaskedStoreCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
return true;
}
+bool clusterSortPtrAccesses(ArrayRef<Value *> VL, ArrayRef<BasicBlock *> BBs,
+ Type *ElemTy, const DataLayout &DL,
+ ScalarEvolution &SE, unsigned MaxDepth,
+ SmallVectorImpl<unsigned> &SortedIndices) {
+ assert(
+ all_of(VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
+ "Expected list of pointer operands.");
+ // Map from bases to a vector of (Ptr, Offset, OrigIdx), which we insert each
+ // Ptr into, sort and return the sorted indices with values next to one
+ // another.
+ SmallMapVector<
+ std::pair<BasicBlock *, Value *>,
+ SmallVector<SmallVector<std::tuple<Value *, int64_t, unsigned>>>, 8>
+ Bases;
+ Bases
+ .try_emplace(std::make_pair(BBs.front(),
+ getUnderlyingObject(VL.front(), MaxDepth)))
+ .first->second.emplace_back()
+ .emplace_back(VL.front(), 0U, 0U);
+
+ SortedIndices.clear();
+ for (auto [Cnt, Ptr] : enumerate(VL.drop_front())) {
+ auto Key = std::make_pair(BBs[Cnt + 1], getUnderlyingObject(Ptr, MaxDepth));
+ bool Found = any_of(Bases.try_emplace(Key).first->second,
+ [&, &Cnt = Cnt, &Ptr = Ptr](auto &Base) {
+ std::optional<int64_t> Diff =
+ getPointersDiff(ElemTy, std::get<0>(Base.front()),
+ ElemTy, Ptr, DL, SE,
+ /*StrictCheck=*/true);
+ if (!Diff)
+ return false;
+
+ Base.emplace_back(Ptr, *Diff, Cnt + 1);
+ return true;
+ });
+
+ if (!Found) {
+ // If we haven't found enough to usefully cluster, return early.
+ if (Bases.size() > VL.size() / 2 - 1)
+ return false;
+
+ // Not found already - add a new Base
+ Bases.find(Key)->second.emplace_back().emplace_back(Ptr, 0, Cnt + 1);
+ }
+ }
+
+ if (Bases.size() == VL.size())
+ return false;
+
+ if (Bases.size() == 1 && (Bases.front().second.size() == 1 ||
+ Bases.front().second.size() == VL.size()))
+ return false;
+
+ // For each of the bases sort the pointers by Offset and check if any of the
+ // base become consecutively allocated.
+ auto ComparePointers = [MaxDepth](Value *Ptr1, Value *Ptr2) {
+ SmallPtrSet<Value *, 13> FirstPointers;
+ SmallPtrSet<Value *, 13> SecondPointers;
+ Value *P1 = Ptr1;
+ Value *P2 = Ptr2;
+ unsigned Depth = 0;
+ while (!FirstPointers.contains(P2) && !SecondPointers.contains(P1)) {
+ if (P1 == P2 || Depth > MaxDepth)
+ return false;
+ FirstPointers.insert(P1);
+ SecondPointers.insert(P2);
+ P1 = getUnderlyingObject(P1, /*MaxLookup=*/1);
+ P2 = getUnderlyingObject(P2, /*MaxLookup=*/1);
+ ++Depth;
+ }
+ assert((FirstPointers.contains(P2) || SecondPointers.contains(P1)) &&
+ "Unable to find matching root.");
+ return FirstPointers.contains(P2) && !SecondPointers.contains(P1);
+ };
+ for (auto &Base : Bases) {
+ for (auto &Vec : Base.second) {
+ if (Vec.size() > 1) {
+ stable_sort(Vec, llvm::less_second());
+ int64_t InitialOffset = std::get<1>(Vec[0]);
+ bool AnyConsecutive =
+ all_of(enumerate(Vec), [InitialOffset](const auto &P) {
+ return std::get<1>(P.value()) ==
+ int64_t(P.index()) + InitialOffset;
+ });
+ // Fill SortedIndices array only if it looks worth-while to sort the
+ // ptrs.
+ if (!AnyConsecutive)
+ return false;
+ }
+ }
+ stable_sort(Base.second, [&](const auto &V1, const auto &V2) {
+ return ComparePointers(std::get<0>(V1.front()), std::get<0>(V2.front()));
+ });
+ }
+
+ for (auto &T : Bases)
+ for (const auto &Vec : T.second)
+ for (const auto &P : Vec)
+ SortedIndices.push_back(std::get<2>(P));
+
+ assert(SortedIndices.size() == VL.size() &&
+ "Expected SortedIndices to be the size of VL");
+ return true;
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
index fa2f5589345d7..d6cc75c3186e2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -21,6 +21,7 @@
namespace llvm {
class AssumptionCache;
+class BasicBlock;
class DataLayout;
class DominatorTree;
class FixedVectorType;
@@ -90,6 +91,14 @@ bool isMaskedStoreCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
SmallVectorImpl<int> &ReuseShuffleIndices,
FixedVectorType *&StoreVecTy);
+/// Clusters \p VL pointers by underlying object and sorts each cluster by
+/// offset, producing \p SortedIndices when the accesses are consecutive.
+/// \p MaxDepth is the recursion limit for getUnderlyingObject.
+bool clusterSortPtrAccesses(ArrayRef<Value *> VL, ArrayRef<BasicBlock *> BBs,
+ Type *ElemTy, const DataLayout &DL,
+ ScalarEvolution &SE, unsigned MaxDepth,
+ SmallVectorImpl<unsigned> &SortedIndices);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPMEMORYUTILS_H
``````````
</details>
https://github.com/llvm/llvm-project/pull/222235
More information about the llvm-commits
mailing list