[llvm] f1f0f21 - [SLP][modularisation][NFC] Move pointer/stride helpers to SLPMemoryUtils (#221786)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 07:48:39 PDT 2026
Author: Madhur Amilkanthwar
Date: 2026-09-08T20:18:32+05:30
New Revision: f1f0f21a51d64d9c0272959f18ece36e3247083e
URL: https://github.com/llvm/llvm-project/commit/f1f0f21a51d64d9c0272959f18ece36e3247083e
DIFF: https://github.com/llvm/llvm-project/commit/f1f0f21a51d64d9c0272959f18ece36e3247083e.diff
LOG: [SLP][modularisation][NFC] Move pointer/stride helpers to SLPMemoryUtils (#221786)
Move the following BoUpSLP-independent pointer/stride helpers out of
SLPVectorizer.cpp into a new SLPVectorizer/SLPMemoryUtils.{h,cpp}:
arePointersCompatible
computeCommonAlignment
calculateRtStride
arePointersCompatible reads the file-local RecursionMaxDepth cl::opt.
The option stays static in SLPVectorizer.cpp; the moved helper takes its
value as an explicit unsigned parameter. Behavior is unchanged.
Part of the SLPVectorizer.cpp modularization effort:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
Assisted by AI.
Added:
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
Modified:
llvm/lib/Transforms/Vectorize/CMakeLists.txt
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 11559b481394a..7334830c422f8 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -25,6 +25,7 @@ add_llvm_component_library(LLVMVectorize
SandboxVectorizer/VecUtils.cpp
SLPVectorizer/SLPCompatibilityAnalysis.cpp
SLPVectorizer/SLPCostAnalysis.cpp
+ SLPVectorizer/SLPMemoryUtils.cpp
SLPVectorizer/SLPTypeUtils.cpp
SLPVectorizer/SLPUtils.cpp
SLPVectorizer.cpp
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e67581f7b11dc..950297eb31cc4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19,6 +19,7 @@
#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "SLPVectorizer/SLPCompatibilityAnalysis.h"
#include "SLPVectorizer/SLPCostAnalysis.h"
+#include "SLPVectorizer/SLPMemoryUtils.h"
#include "SLPVectorizer/SLPTypeUtils.h"
#include "SLPVectorizer/SLPUtils.h"
#include "llvm/ADT/DenseMap.h"
@@ -5699,144 +5700,6 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
return std::move(CurrentOrder);
}
-static bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
- const TargetLibraryInfo &TLI,
- bool CompareOpcodes = true) {
- if (getUnderlyingObject(Ptr1, RecursionMaxDepth) !=
- getUnderlyingObject(Ptr2, RecursionMaxDepth))
- return false;
- auto *GEP1 = dyn_cast<GetElementPtrInst>(Ptr1);
- auto *GEP2 = dyn_cast<GetElementPtrInst>(Ptr2);
- return (!GEP1 || GEP1->getNumOperands() == 2) &&
- (!GEP2 || GEP2->getNumOperands() == 2) &&
- (((!GEP1 || isConstant(GEP1->getOperand(1))) &&
- (!GEP2 || isConstant(GEP2->getOperand(1)))) ||
- !CompareOpcodes ||
- (GEP1 && GEP2 &&
- getSameOpcode({GEP1->getOperand(1), GEP2->getOperand(1)}, TLI)));
-}
-
-/// Calculates minimal alignment as a common alignment.
-template <typename T>
-static Align computeCommonAlignment(ArrayRef<Value *> VL) {
- Align CommonAlignment = cast<T>(VL.consume_front())->getAlign();
- for (Value *V : VL)
- CommonAlignment = std::min(CommonAlignment, cast<T>(V)->getAlign());
- return CommonAlignment;
-}
-
-/// Checks if the provided list of pointers \p Pointers represents the strided
-/// pointers for type ElemTy. If they are not, nullptr is returned.
-/// Otherwise, SCEV* of the stride value is returned.
-/// If `PointerOps` can be rearanged into the following sequence:
-/// ```
-/// %x + c_0 * stride,
-/// %x + c_1 * stride,
-/// %x + c_2 * stride
-/// ...
-/// ```
-/// where each `c_i` is constant. The SCEV of the `stride` will be returned.
-static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
- const DataLayout &DL, ScalarEvolution &SE,
- SmallVectorImpl<unsigned> &SortedIndices) {
- SmallVector<const SCEV *> SCEVs;
- const SCEV *PtrSCEVLowest = nullptr;
- const SCEV *PtrSCEVHighest = nullptr;
- // Find lower/upper pointers from the PointerOps (i.e. with lowest and highest
- // addresses).
- for (Value *Ptr : PointerOps) {
- const SCEV *PtrSCEV = SE.getSCEV(Ptr);
- if (!PtrSCEV)
- return nullptr;
- SCEVs.push_back(PtrSCEV);
- if (!PtrSCEVLowest && !PtrSCEVHighest) {
- PtrSCEVLowest = PtrSCEVHighest = PtrSCEV;
- continue;
- }
- const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
- if (isa<SCEVCouldNotCompute>(Diff))
- return nullptr;
- if (Diff->isNonConstantNegative()) {
- PtrSCEVLowest = PtrSCEV;
- continue;
- }
- const SCEV *Diff1 = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEV);
- if (isa<SCEVCouldNotCompute>(Diff1))
- return nullptr;
- if (Diff1->isNonConstantNegative()) {
- PtrSCEVHighest = PtrSCEV;
- continue;
- }
- }
- // Dist = PtrSCEVHighest - PtrSCEVLowest;
- const SCEV *Dist = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEVLowest);
- if (isa<SCEVCouldNotCompute>(Dist))
- return nullptr;
- int Size = DL.getTypeStoreSize(ElemTy);
- auto TryGetStride = [&](const SCEV *Dist,
- const SCEV *Multiplier) -> const SCEV * {
- if (const auto *M = dyn_cast<SCEVMulExpr>(Dist)) {
- if (M->getOperand(0) == Multiplier)
- return M->getOperand(1);
- if (M->getOperand(1) == Multiplier)
- return M->getOperand(0);
- return nullptr;
- }
- if (Multiplier == Dist)
- return SE.getConstant(Dist->getType(), 1);
- return SE.getUDivExactExpr(Dist, Multiplier);
- };
- // Stride_in_elements = Dist / element_size * (num_elems - 1).
- const SCEV *Stride = nullptr;
- if (Size != 1 || SCEVs.size() > 1) {
- const SCEV *Sz = SE.getConstant(Dist->getType(), Size * (SCEVs.size() - 1));
- Stride = TryGetStride(Dist, Sz);
- if (!Stride)
- return nullptr;
- }
- if (!Stride || isa<SCEVConstant>(Stride))
- return nullptr;
- // Iterate through all pointers and check if all distances are
- // unique multiple of Stride.
- using DistOrdPair = std::pair<int64_t, int>;
- auto Compare = llvm::less_first();
- std::set<DistOrdPair, decltype(Compare)> Offsets(Compare);
- bool IsConsecutive = true;
- for (const auto [Idx, PtrSCEV] : enumerate(SCEVs)) {
- unsigned Dist = 0;
- if (PtrSCEV != PtrSCEVLowest) {
- const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
- const SCEV *Coeff = TryGetStride(Diff, Stride);
- if (!Coeff)
- return nullptr;
- const auto *SC = dyn_cast<SCEVConstant>(Coeff);
- if (!SC || isa<SCEVCouldNotCompute>(SC))
- return nullptr;
- if (!SE.getMinusSCEV(PtrSCEV, SE.getAddExpr(PtrSCEVLowest,
- SE.getMulExpr(Stride, SC)))
- ->isZero())
- return nullptr;
- Dist = SC->getAPInt().getZExtValue();
- }
- // If the strides are not the same or repeated, we can't vectorize.
- if ((Dist / Size) * Size != Dist || (Dist / Size) >= SCEVs.size())
- return nullptr;
- auto Res = Offsets.emplace(Dist, Idx);
- if (!Res.second)
- return nullptr;
- // Consecutive order if the inserted element is the last one.
- IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end();
- }
- SortedIndices.clear();
- if (!IsConsecutive) {
- // Fill SortedIndices array only if it is non-consecutive.
- SortedIndices.resize(PointerOps.size());
- for (const auto [Idx, Pair] : enumerate(Offsets))
- SortedIndices[Idx] = Pair.second;
- }
- return Stride;
-}
-
/// Builds compress-like mask for shuffles for the given \p PointerOps, ordered
/// with \p Order.
/// \return true if the mask represents strided access, false - otherwise.
@@ -6527,7 +6390,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
return LoadsState::Gather;
if (!all_of(PointerOps, [&](Value *P) {
- return arePointersCompatible(P, PointerOps.front(), *TLI);
+ return arePointersCompatible(P, PointerOps.front(), *TLI,
+ RecursionMaxDepth);
}))
return LoadsState::Gather;
@@ -14047,7 +13911,8 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
}
for (LoadInst *RLI : LIt->second) {
if (arePointersCompatible(RLI->getPointerOperand(),
- LI->getPointerOperand(), *TLI)) {
+ LI->getPointerOperand(), *TLI,
+ RecursionMaxDepth)) {
hash_code SubKey = hash_value(RLI->getPointerOperand());
return SubKey;
}
@@ -30610,7 +30475,8 @@ class HorizontalReduction {
}
for (LoadInst *RLI : LIt->second) {
if (arePointersCompatible(RLI->getPointerOperand(),
- LI->getPointerOperand(), TLI)) {
+ LI->getPointerOperand(), TLI,
+ RecursionMaxDepth)) {
hash_code SubKey = hash_value(RLI->getPointerOperand());
return SubKey;
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
new file mode 100644
index 0000000000000..7b2d5d1dd34ef
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.cpp
@@ -0,0 +1,157 @@
+//===- SLPMemoryUtils.cpp - SLP pointer/stride helpers --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SLPMemoryUtils.h"
+#include "SLPCompatibilityAnalysis.h"
+#include "SLPUtils.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+
+#include <algorithm>
+#include <set>
+#include <utility>
+
+using namespace llvm;
+
+namespace llvm::slpvectorizer {
+
+bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
+ const TargetLibraryInfo &TLI, unsigned MaxDepth,
+ bool CompareOpcodes) {
+ if (getUnderlyingObject(Ptr1, MaxDepth) !=
+ getUnderlyingObject(Ptr2, MaxDepth))
+ return false;
+ auto *GEP1 = dyn_cast<GetElementPtrInst>(Ptr1);
+ auto *GEP2 = dyn_cast<GetElementPtrInst>(Ptr2);
+ return (!GEP1 || GEP1->getNumOperands() == 2) &&
+ (!GEP2 || GEP2->getNumOperands() == 2) &&
+ (((!GEP1 || isConstant(GEP1->getOperand(1))) &&
+ (!GEP2 || isConstant(GEP2->getOperand(1)))) ||
+ !CompareOpcodes ||
+ (GEP1 && GEP2 &&
+ getSameOpcode({GEP1->getOperand(1), GEP2->getOperand(1)}, TLI)));
+}
+
+/// Calculates minimal alignment as a common alignment.
+template <typename T> Align computeCommonAlignment(ArrayRef<Value *> VL) {
+ Align CommonAlignment = cast<T>(VL.consume_front())->getAlign();
+ for (Value *V : VL)
+ CommonAlignment = std::min(CommonAlignment, cast<T>(V)->getAlign());
+ return CommonAlignment;
+}
+
+template Align computeCommonAlignment<LoadInst>(ArrayRef<Value *>);
+template Align computeCommonAlignment<StoreInst>(ArrayRef<Value *>);
+
+const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
+ const DataLayout &DL, ScalarEvolution &SE,
+ SmallVectorImpl<unsigned> &SortedIndices) {
+ SmallVector<const SCEV *> SCEVs;
+ const SCEV *PtrSCEVLowest = nullptr;
+ const SCEV *PtrSCEVHighest = nullptr;
+ // Find lower/upper pointers from the PointerOps (i.e. with lowest and highest
+ // addresses).
+ for (Value *Ptr : PointerOps) {
+ const SCEV *PtrSCEV = SE.getSCEV(Ptr);
+ if (!PtrSCEV)
+ return nullptr;
+ SCEVs.push_back(PtrSCEV);
+ if (!PtrSCEVLowest && !PtrSCEVHighest) {
+ PtrSCEVLowest = PtrSCEVHighest = PtrSCEV;
+ continue;
+ }
+ const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
+ if (isa<SCEVCouldNotCompute>(Diff))
+ return nullptr;
+ if (Diff->isNonConstantNegative()) {
+ PtrSCEVLowest = PtrSCEV;
+ continue;
+ }
+ const SCEV *Diff1 = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEV);
+ if (isa<SCEVCouldNotCompute>(Diff1))
+ return nullptr;
+ if (Diff1->isNonConstantNegative()) {
+ PtrSCEVHighest = PtrSCEV;
+ continue;
+ }
+ }
+ // Dist = PtrSCEVHighest - PtrSCEVLowest;
+ const SCEV *Dist = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEVLowest);
+ if (isa<SCEVCouldNotCompute>(Dist))
+ return nullptr;
+ int Size = DL.getTypeStoreSize(ElemTy);
+ auto TryGetStride = [&](const SCEV *Dist,
+ const SCEV *Multiplier) -> const SCEV * {
+ if (const auto *M = dyn_cast<SCEVMulExpr>(Dist)) {
+ if (M->getOperand(0) == Multiplier)
+ return M->getOperand(1);
+ if (M->getOperand(1) == Multiplier)
+ return M->getOperand(0);
+ return nullptr;
+ }
+ if (Multiplier == Dist)
+ return SE.getConstant(Dist->getType(), 1);
+ return SE.getUDivExactExpr(Dist, Multiplier);
+ };
+ // Stride_in_elements = Dist / element_size * (num_elems - 1).
+ const SCEV *Stride = nullptr;
+ if (Size != 1 || SCEVs.size() > 1) {
+ const SCEV *Sz = SE.getConstant(Dist->getType(), Size * (SCEVs.size() - 1));
+ Stride = TryGetStride(Dist, Sz);
+ if (!Stride)
+ return nullptr;
+ }
+ if (!Stride || isa<SCEVConstant>(Stride))
+ return nullptr;
+ // Iterate through all pointers and check if all distances are
+ // unique multiple of Stride.
+ using DistOrdPair = std::pair<int64_t, int>;
+ auto Compare = llvm::less_first();
+ std::set<DistOrdPair, decltype(Compare)> Offsets(Compare);
+ bool IsConsecutive = true;
+ for (const auto [Idx, PtrSCEV] : enumerate(SCEVs)) {
+ unsigned Dist = 0;
+ if (PtrSCEV != PtrSCEVLowest) {
+ const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
+ const SCEV *Coeff = TryGetStride(Diff, Stride);
+ if (!Coeff)
+ return nullptr;
+ const auto *SC = dyn_cast<SCEVConstant>(Coeff);
+ if (!SC || isa<SCEVCouldNotCompute>(SC))
+ return nullptr;
+ if (!SE.getMinusSCEV(PtrSCEV, SE.getAddExpr(PtrSCEVLowest,
+ SE.getMulExpr(Stride, SC)))
+ ->isZero())
+ return nullptr;
+ Dist = SC->getAPInt().getZExtValue();
+ }
+ // If the strides are not the same or repeated, we can't vectorize.
+ if ((Dist / Size) * Size != Dist || (Dist / Size) >= SCEVs.size())
+ return nullptr;
+ auto Res = Offsets.emplace(Dist, Idx);
+ if (!Res.second)
+ return nullptr;
+ // Consecutive order if the inserted element is the last one.
+ IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end();
+ }
+ SortedIndices.clear();
+ if (!IsConsecutive) {
+ // Fill SortedIndices array only if it is non-consecutive.
+ SortedIndices.resize(PointerOps.size());
+ for (const auto [Idx, Pair] : enumerate(Offsets))
+ SortedIndices[Idx] = Pair.second;
+ }
+ return Stride;
+}
+
+} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
new file mode 100644
index 0000000000000..75561b0b7b40b
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPMemoryUtils.h
@@ -0,0 +1,56 @@
+//===- SLPMemoryUtils.h - SLP pointer/stride helpers -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Internal header used by SLPVectorizer.cpp. It declares free pointer and
+// stride helpers that do not depend on BoUpSLP or any other SLP-private type.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPMEMORYUTILS_H
+#define LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPMEMORYUTILS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Alignment.h"
+
+namespace llvm {
+class DataLayout;
+class SCEV;
+class ScalarEvolution;
+class TargetLibraryInfo;
+class Type;
+class Value;
+} // namespace llvm
+
+namespace llvm::slpvectorizer {
+
+/// \p MaxDepth is the recursion limit for getUnderlyingObject.
+bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
+ const TargetLibraryInfo &TLI, unsigned MaxDepth,
+ bool CompareOpcodes = true);
+
+/// Calculates minimal alignment as a common alignment.
+template <typename T> Align computeCommonAlignment(ArrayRef<Value *> VL);
+
+/// Checks if the provided list of pointers \p Pointers represents the strided
+/// pointers for type ElemTy. If they are not, nullptr is returned.
+/// Otherwise, SCEV* of the stride value is returned.
+/// If `PointerOps` can be rearranged into the following sequence:
+/// ```
+/// %x + c_0 * stride,
+/// %x + c_1 * stride,
+/// %x + c_2 * stride
+/// ...
+/// ```
+/// where each `c_i` is constant. The SCEV of the `stride` will be returned.
+const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
+ const DataLayout &DL, ScalarEvolution &SE,
+ SmallVectorImpl<unsigned> &SortedIndices);
+
+} // namespace llvm::slpvectorizer
+
+#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPMEMORYUTILS_H
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 4c9c8c72d3dac..3a6fce2c029e9 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -16,6 +16,7 @@ static_library("Vectorize") {
"SLPVectorizer.cpp",
"SLPVectorizer/SLPCompatibilityAnalysis.cpp",
"SLPVectorizer/SLPCostAnalysis.cpp",
+ "SLPVectorizer/SLPMemoryUtils.cpp",
"SLPVectorizer/SLPTypeUtils.cpp",
"SLPVectorizer/SLPUtils.cpp",
"SandboxVectorizer/DependencyGraph.cpp",
More information about the llvm-commits
mailing list