[llvm] 4acc5f1 - [SLP][modularisation][NFC] Move clusterSortPtrAccesses to SLPMemoryUtils (#222235)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 21:07:00 PDT 2026
Author: Madhur Amilkanthwar
Date: 2026-09-10T04:06:54Z
New Revision: 4acc5f1ff8d181bb948a60b7eb2a6371e241051e
URL: https://github.com/llvm/llvm-project/commit/4acc5f1ff8d181bb948a60b7eb2a6371e241051e
DIFF: https://github.com/llvm/llvm-project/commit/4acc5f1ff8d181bb948a60b7eb2a6371e241051e.diff
LOG: [SLP][modularisation][NFC] Move clusterSortPtrAccesses to SLPMemoryUtils (#222235)
Move the BoUpSLP-independent pointer-clustering sort out of
SLPVectorizer.cpp into SLPVectorizer/SLPMemoryUtils.{h,cpp}.
clusterSortPtrAccesses reads the file-local RecursionMaxDepth cl::opt
via getUnderlyingObject; the option stays static in SLPVectorizer.cpp
and the moved helper takes its value as an explicit unsigned parameter
(also captured by the local comparison lambda). Behavior is unchanged.
Part of the SLPVectorizer.cpp modularization effort:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
Assisted by AI
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9cc4eac5f8752..681aa5da811e3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6439,111 +6439,6 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
return LoadsState::Gather;
}
-static bool clusterSortPtrAccesses(ArrayRef<Value *> VL,
- ArrayRef<BasicBlock *> BBs, Type *ElemTy,
- const DataLayout &DL, ScalarEvolution &SE,
- SmallVectorImpl<unsigned> &SortedIndices) {
- assert(
- all_of(VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
- "Expected list of pointer operands.");
- // Map from bases to a vector of (Ptr, Offset, OrigIdx), which we insert each
- // Ptr into, sort and return the sorted indices with values next to one
- // another.
- SmallMapVector<
- std::pair<BasicBlock *, Value *>,
- SmallVector<SmallVector<std::tuple<Value *, int64_t, unsigned>>>, 8>
- Bases;
- Bases
- .try_emplace(std::make_pair(
- BBs.front(), getUnderlyingObject(VL.front(), RecursionMaxDepth)))
- .first->second.emplace_back().emplace_back(VL.front(), 0U, 0U);
-
- SortedIndices.clear();
- for (auto [Cnt, Ptr] : enumerate(VL.drop_front())) {
- auto Key = std::make_pair(BBs[Cnt + 1],
- getUnderlyingObject(Ptr, RecursionMaxDepth));
- bool Found = any_of(Bases.try_emplace(Key).first->second,
- [&, &Cnt = Cnt, &Ptr = Ptr](auto &Base) {
- std::optional<int64_t> Diff =
- getPointersDiff(ElemTy, std::get<0>(Base.front()),
- ElemTy, Ptr, DL, SE,
- /*StrictCheck=*/true);
- if (!Diff)
- return false;
-
- Base.emplace_back(Ptr, *Diff, Cnt + 1);
- return true;
- });
-
- if (!Found) {
- // If we haven't found enough to usefully cluster, return early.
- if (Bases.size() > VL.size() / 2 - 1)
- return false;
-
- // Not found already - add a new Base
- Bases.find(Key)->second.emplace_back().emplace_back(Ptr, 0, Cnt + 1);
- }
- }
-
- if (Bases.size() == VL.size())
- return false;
-
- if (Bases.size() == 1 && (Bases.front().second.size() == 1 ||
- Bases.front().second.size() == VL.size()))
- return false;
-
- // For each of the bases sort the pointers by Offset and check if any of the
- // base become consecutively allocated.
- auto ComparePointers = [](Value *Ptr1, Value *Ptr2) {
- SmallPtrSet<Value *, 13> FirstPointers;
- SmallPtrSet<Value *, 13> SecondPointers;
- Value *P1 = Ptr1;
- Value *P2 = Ptr2;
- unsigned Depth = 0;
- while (!FirstPointers.contains(P2) && !SecondPointers.contains(P1)) {
- if (P1 == P2 || Depth > RecursionMaxDepth)
- return false;
- FirstPointers.insert(P1);
- SecondPointers.insert(P2);
- P1 = getUnderlyingObject(P1, /*MaxLookup=*/1);
- P2 = getUnderlyingObject(P2, /*MaxLookup=*/1);
- ++Depth;
- }
- assert((FirstPointers.contains(P2) || SecondPointers.contains(P1)) &&
- "Unable to find matching root.");
- return FirstPointers.contains(P2) && !SecondPointers.contains(P1);
- };
- for (auto &Base : Bases) {
- for (auto &Vec : Base.second) {
- if (Vec.size() > 1) {
- stable_sort(Vec, llvm::less_second());
- int64_t InitialOffset = std::get<1>(Vec[0]);
- bool AnyConsecutive =
- all_of(enumerate(Vec), [InitialOffset](const auto &P) {
- return std::get<1>(P.value()) ==
- int64_t(P.index()) + InitialOffset;
- });
- // Fill SortedIndices array only if it looks worth-while to sort the
- // ptrs.
- if (!AnyConsecutive)
- return false;
- }
- }
- stable_sort(Base.second, [&](const auto &V1, const auto &V2) {
- return ComparePointers(std::get<0>(V1.front()), std::get<0>(V2.front()));
- });
- }
-
- for (auto &T : Bases)
- for (const auto &Vec : T.second)
- for (const auto &P : Vec)
- SortedIndices.push_back(std::get<2>(P));
-
- assert(SortedIndices.size() == VL.size() &&
- "Expected SortedIndices to be the size of VL");
- return true;
-}
-
std::optional<BoUpSLP::OrdersType>
BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
assert(TE.isGather() && "Expected gather node only.");
@@ -6563,7 +6458,8 @@ BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
BoUpSLP::OrdersType Order;
if (!LoadEntriesToVectorize.contains(TE.Idx) &&
- clusterSortPtrAccesses(Ptrs, BBs, ScalarTy, *DL, *SE, Order))
+ clusterSortPtrAccesses(Ptrs, BBs, ScalarTy, *DL, *SE, RecursionMaxDepth,
+ Order))
return std::move(Order);
return std::nullopt;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
index ced6baf782628..382adb71cede2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -13,8 +13,10 @@
#include "SLPUtils.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/ScalarEvolution.h"
@@ -31,6 +33,7 @@
#include <limits>
#include <optional>
#include <set>
+#include <tuple>
#include <utility>
using namespace llvm;
@@ -409,4 +412,109 @@ bool isMaskedStoreCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
return true;
}
+bool clusterSortPtrAccesses(ArrayRef<Value *> VL, ArrayRef<BasicBlock *> BBs,
+ Type *ElemTy, const DataLayout &DL,
+ ScalarEvolution &SE, unsigned MaxDepth,
+ SmallVectorImpl<unsigned> &SortedIndices) {
+ assert(
+ all_of(VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
+ "Expected list of pointer operands.");
+ // Map from bases to a vector of (Ptr, Offset, OrigIdx), which we insert each
+ // Ptr into, sort and return the sorted indices with values next to one
+ // another.
+ SmallMapVector<
+ std::pair<BasicBlock *, Value *>,
+ SmallVector<SmallVector<std::tuple<Value *, int64_t, unsigned>>>, 8>
+ Bases;
+ Bases
+ .try_emplace(std::make_pair(BBs.front(),
+ getUnderlyingObject(VL.front(), MaxDepth)))
+ .first->second.emplace_back()
+ .emplace_back(VL.front(), 0U, 0U);
+
+ SortedIndices.clear();
+ for (auto [Cnt, Ptr] : enumerate(VL.drop_front())) {
+ auto Key = std::make_pair(BBs[Cnt + 1], getUnderlyingObject(Ptr, MaxDepth));
+ bool Found = any_of(Bases.try_emplace(Key).first->second,
+ [&, &Cnt = Cnt, &Ptr = Ptr](auto &Base) {
+ std::optional<int64_t> Diff =
+ getPointersDiff(ElemTy, std::get<0>(Base.front()),
+ ElemTy, Ptr, DL, SE,
+ /*StrictCheck=*/true);
+ if (!Diff)
+ return false;
+
+ Base.emplace_back(Ptr, *Diff, Cnt + 1);
+ return true;
+ });
+
+ if (!Found) {
+ // If we haven't found enough to usefully cluster, return early.
+ if (Bases.size() > VL.size() / 2 - 1)
+ return false;
+
+ // Not found already - add a new Base
+ Bases.find(Key)->second.emplace_back().emplace_back(Ptr, 0, Cnt + 1);
+ }
+ }
+
+ if (Bases.size() == VL.size())
+ return false;
+
+ if (Bases.size() == 1 && (Bases.front().second.size() == 1 ||
+ Bases.front().second.size() == VL.size()))
+ return false;
+
+ // For each of the bases sort the pointers by Offset and check if any of the
+ // base become consecutively allocated.
+ auto ComparePointers = [MaxDepth](Value *Ptr1, Value *Ptr2) {
+ SmallPtrSet<Value *, 13> FirstPointers;
+ SmallPtrSet<Value *, 13> SecondPointers;
+ Value *P1 = Ptr1;
+ Value *P2 = Ptr2;
+ unsigned Depth = 0;
+ while (!FirstPointers.contains(P2) && !SecondPointers.contains(P1)) {
+ if (P1 == P2 || Depth > MaxDepth)
+ return false;
+ FirstPointers.insert(P1);
+ SecondPointers.insert(P2);
+ P1 = getUnderlyingObject(P1, /*MaxLookup=*/1);
+ P2 = getUnderlyingObject(P2, /*MaxLookup=*/1);
+ ++Depth;
+ }
+ assert((FirstPointers.contains(P2) || SecondPointers.contains(P1)) &&
+ "Unable to find matching root.");
+ return FirstPointers.contains(P2) && !SecondPointers.contains(P1);
+ };
+ for (auto &Base : Bases) {
+ for (auto &Vec : Base.second) {
+ if (Vec.size() > 1) {
+ stable_sort(Vec, llvm::less_second());
+ int64_t InitialOffset = std::get<1>(Vec[0]);
+ bool AnyConsecutive =
+ all_of(enumerate(Vec), [InitialOffset](const auto &P) {
+ return std::get<1>(P.value()) ==
+ int64_t(P.index()) + InitialOffset;
+ });
+ // Fill SortedIndices array only if it looks worth-while to sort the
+ // ptrs.
+ if (!AnyConsecutive)
+ return false;
+ }
+ }
+ stable_sort(Base.second, [&](const auto &V1, const auto &V2) {
+ return ComparePointers(std::get<0>(V1.front()), std::get<0>(V2.front()));
+ });
+ }
+
+ for (auto &T : Bases)
+ for (const auto &Vec : T.second)
+ for (const auto &P : Vec)
+ SortedIndices.push_back(std::get<2>(P));
+
+ assert(SortedIndices.size() == VL.size() &&
+ "Expected SortedIndices to be the size of VL");
+ return true;
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
index fa2f5589345d7..d1df27e1fb6c3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -21,6 +21,7 @@
namespace llvm {
class AssumptionCache;
+class BasicBlock;
class DataLayout;
class DominatorTree;
class FixedVectorType;
@@ -90,6 +91,14 @@ bool isMaskedStoreCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
SmallVectorImpl<int> &ReuseShuffleIndices,
FixedVectorType *&StoreVecTy);
+/// Clusters \p VL pointers by (basic block, underlying object) pair and sorts
+/// each cluster by offset. Returns false and leaves \p SortedIndices empty if
+/// the accesses are not worth reordering.
+bool clusterSortPtrAccesses(ArrayRef<Value *> VL, ArrayRef<BasicBlock *> BBs,
+ Type *ElemTy, const DataLayout &DL,
+ ScalarEvolution &SE, unsigned MaxDepth,
+ SmallVectorImpl<unsigned> &SortedIndices);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPMEMORYUTILS_H
More information about the llvm-commits
mailing list