[llvm] [SLPVectorizer] Widen strided loads. (PR #153074)
Mikhail Gudim via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 13 13:23:47 PDT 2025
https://github.com/mgudim updated https://github.com/llvm/llvm-project/pull/153074
>From 6bf9092c2609576384851fb248558fdd44a656d8 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Fri, 8 Aug 2025 07:52:29 -0700
Subject: [PATCH 01/12] [SLPVectorizer] Widen strided loads.
Currently SLPVectorizer can generate strided loads only for this
pattern:
```
load %base + 0 * %stride
load %base + 1 * %stride
...
load %base + n * %stride
```
In this PR we extend it to this pattern:
```
; load w consecutive elements starting at %base
load %base + 0 * %stride + 0
load %base + 0 * %stride + 1
load %base + 0 * %stride + 2
...
load %base + 0 * %stride + (w - 1)
; load w consecutive elements starting at %base + 1 * %stride
load %base + 1 * %stride + 0
load %base + 1 * %stride + 1
load %base + 1 * %stride + 2
...
load %base + 1 * %stride + (w - 1)
...
; load w consecutive elements starting at %base + n * %stride
load %base + n * %stride
load %base + n * %stride + 0
load %base + n * %stride + 1
load %base + n * %stride + 2
...
load %base + n * %stride + (w - 1)
```
This works for both run-time and constant strides.
---
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 9 +
.../Target/RISCV/RISCVTargetTransformInfo.h | 2 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 538 +++++++++++++-----
.../SLPVectorizer/RISCV/x264-satd-8x4.ll | 483 ++++++++++++++++
4 files changed, 873 insertions(+), 159 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index af78b3cc2c7ff..4d43cb7ec0300 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -37,6 +37,11 @@ static cl::opt<unsigned> SLPMaxVF(
"exclusively by SLP vectorizer."),
cl::Hidden);
+static cl::opt<bool> SLPPreferAltOpcVectorization(
+ "riscv-v-slp-prefer-alt-opc-vectorization",
+ cl::desc("Controls preferAlternateOpcodeVectorization"), cl::init(false),
+ cl::Hidden);
+
static cl::opt<unsigned>
RVVMinTripCount("riscv-v-min-trip-count",
cl::desc("Set the lower bound of a trip count to decide on "
@@ -3018,3 +3023,7 @@ RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
}
return Options;
}
+
+bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
+ return SLPPreferAltOpcVectorization;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 6a1f4b3e3bedf..254908f97186c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -132,7 +132,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
- bool preferAlternateOpcodeVectorization() const override { return false; }
+ bool preferAlternateOpcodeVectorization() const override;
bool preferEpilogueVectorization() const override {
// Epilogue vectorization is usually unprofitable - tail folding or
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3045eeb3eb48e..207d9fc969f76 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1918,6 +1918,21 @@ class BoUpSLP {
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
+ // If we decide to generate strided load / store, this struct contains all the
+ // necessary info. It's fields are calculated by analyzeRtStrideCandidate and
+ // analyzeConstantStrideCandidate. Note that Stride can be given either as a
+ // SCEV or as a Value if it already exists.
+ // To get the stride in bytes, StrideVal (or value obtained from StrideSCEV)
+ // has to by multiplied by the size of element of FixedVectorType.
+ struct StridedPtrInfo {
+ Value *StrideVal = nullptr;
+ const SCEV *StrideSCEV = nullptr;
+ // Represents the ammount which needs to be added to the base pointer of
+ // strided load.
+ FixedVectorType *Ty = nullptr;
+ };
+ DenseMap<TreeEntry *, StridedPtrInfo> TreeEntryToStridedPtrInfoMap;
+
public:
/// Tracks the state we can represent the loads in the given sequence.
enum class LoadsState {
@@ -2078,6 +2093,7 @@ class BoUpSLP {
UserIgnoreList = nullptr;
PostponedGathers.clear();
ValueToGatherNodes.clear();
+ TreeEntryToStridedPtrInfoMap.clear();
}
unsigned getTreeSize() const { return VectorizableTree.size(); }
@@ -2214,6 +2230,17 @@ class BoUpSLP {
/// may not be necessary.
bool isLoadCombineCandidate(ArrayRef<Value *> Stores) const;
+ bool analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps, Type *ElemTy,
+ Align CommonAlignment,
+ SmallVectorImpl<unsigned> &SortedIndices,
+ StridedPtrInfo *SPtrInfo) const;
+
+ bool analyzeConstantStrideCandidate(ArrayRef<Value *> PointerOps,
+ Type *ElemTy, Align CommonAlignment,
+ SmallVectorImpl<unsigned> &SortedIndices,
+ StridedPtrInfo *SPtrInfo, int64_t Diff,
+ Value *Ptr0, Value *PtrN) const;
+
/// Checks if the given array of loads can be represented as a vectorized,
/// scatter or just simple gather.
/// \param VL list of loads.
@@ -2227,6 +2254,7 @@ class BoUpSLP {
LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
SmallVectorImpl<unsigned> &Order,
SmallVectorImpl<Value *> &PointerOps,
+ StridedPtrInfo *SPtrInfo = nullptr,
unsigned *BestVF = nullptr,
bool TryRecursiveCheck = true) const;
@@ -4467,11 +4495,10 @@ class BoUpSLP {
/// Checks if the specified list of the instructions/values can be vectorized
/// and fills required data before actual scheduling of the instructions.
- TreeEntry::EntryState
- getScalarsVectorizationState(const InstructionsState &S, ArrayRef<Value *> VL,
- bool IsScatterVectorizeUserTE,
- OrdersType &CurrentOrder,
- SmallVectorImpl<Value *> &PointerOps);
+ TreeEntry::EntryState getScalarsVectorizationState(
+ const InstructionsState &S, ArrayRef<Value *> VL,
+ bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo *SPtrInfo);
/// Maps a specific scalar to its tree entry(ies).
SmallDenseMap<Value *, SmallVector<TreeEntry *>> ScalarToTreeEntries;
@@ -6314,18 +6341,12 @@ static bool isReverseOrder(ArrayRef<unsigned> Order) {
});
}
-/// Checks if the provided list of pointers \p Pointers represents the strided
-/// pointers for type ElemTy. If they are not, std::nullopt is returned.
-/// Otherwise, if \p Inst is not specified, just initialized optional value is
-/// returned to show that the pointers represent strided pointers. If \p Inst
-/// specified, the runtime stride is materialized before the given \p Inst.
-/// \returns std::nullopt if the pointers are not pointers with the runtime
-/// stride, nullptr or actual stride value, otherwise.
-static std::optional<Value *>
-calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
- const DataLayout &DL, ScalarEvolution &SE,
- SmallVectorImpl<unsigned> &SortedIndices,
- Instruction *Inst = nullptr) {
+/// Returns a SCEV expression for the stride if PointerOps is a set of strided
+/// pointers, or nullptr otherwise.
+static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
+ const DataLayout &DL, ScalarEvolution &SE,
+ SmallVectorImpl<unsigned> &SortedIndices,
+ SmallVectorImpl<int64_t> &Coeffs) {
SmallVector<const SCEV *> SCEVs;
const SCEV *PtrSCEVLowest = nullptr;
const SCEV *PtrSCEVHighest = nullptr;
@@ -6334,7 +6355,7 @@ calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
for (Value *Ptr : PointerOps) {
const SCEV *PtrSCEV = SE.getSCEV(Ptr);
if (!PtrSCEV)
- return std::nullopt;
+ return nullptr;
SCEVs.push_back(PtrSCEV);
if (!PtrSCEVLowest && !PtrSCEVHighest) {
PtrSCEVLowest = PtrSCEVHighest = PtrSCEV;
@@ -6342,14 +6363,14 @@ calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
}
const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
if (isa<SCEVCouldNotCompute>(Diff))
- return std::nullopt;
+ return nullptr;
if (Diff->isNonConstantNegative()) {
PtrSCEVLowest = PtrSCEV;
continue;
}
const SCEV *Diff1 = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEV);
if (isa<SCEVCouldNotCompute>(Diff1))
- return std::nullopt;
+ return nullptr;
if (Diff1->isNonConstantNegative()) {
PtrSCEVHighest = PtrSCEV;
continue;
@@ -6358,7 +6379,7 @@ calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
// Dist = PtrSCEVHighest - PtrSCEVLowest;
const SCEV *Dist = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEVLowest);
if (isa<SCEVCouldNotCompute>(Dist))
- return std::nullopt;
+ return nullptr;
int Size = DL.getTypeStoreSize(ElemTy);
auto TryGetStride = [&](const SCEV *Dist,
const SCEV *Multiplier) -> const SCEV * {
@@ -6379,10 +6400,10 @@ calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
const SCEV *Sz = SE.getConstant(Dist->getType(), Size * (SCEVs.size() - 1));
Stride = TryGetStride(Dist, Sz);
if (!Stride)
- return std::nullopt;
+ return nullptr;
}
if (!Stride || isa<SCEVConstant>(Stride))
- return std::nullopt;
+ return nullptr;
// Iterate through all pointers and check if all distances are
// unique multiple of Stride.
using DistOrdPair = std::pair<int64_t, int>;
@@ -6396,42 +6417,184 @@ calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
const SCEV *Coeff = TryGetStride(Diff, Stride);
if (!Coeff)
- return std::nullopt;
+ return nullptr;
const auto *SC = dyn_cast<SCEVConstant>(Coeff);
if (!SC || isa<SCEVCouldNotCompute>(SC))
- return std::nullopt;
+ return nullptr;
+ Coeffs.push_back((int64_t)SC->getAPInt().getLimitedValue());
if (!SE.getMinusSCEV(PtrSCEV, SE.getAddExpr(PtrSCEVLowest,
SE.getMulExpr(Stride, SC)))
->isZero())
- return std::nullopt;
+ return nullptr;
Dist = SC->getAPInt().getZExtValue();
- }
+ } else
+ Coeffs.push_back(0);
// If the strides are not the same or repeated, we can't vectorize.
if ((Dist / Size) * Size != Dist || (Dist / Size) >= SCEVs.size())
- return std::nullopt;
+ return nullptr;
auto Res = Offsets.emplace(Dist, Cnt);
if (!Res.second)
- return std::nullopt;
+ return nullptr;
// Consecutive order if the inserted element is the last one.
IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end();
++Cnt;
}
if (Offsets.size() != SCEVs.size())
- return std::nullopt;
+ return nullptr;
SortedIndices.clear();
- if (!IsConsecutive) {
- // Fill SortedIndices array only if it is non-consecutive.
- SortedIndices.resize(PointerOps.size());
- Cnt = 0;
- for (const std::pair<int64_t, int> &Pair : Offsets) {
- SortedIndices[Cnt] = Pair.second;
- ++Cnt;
+ SortedIndices.resize(PointerOps.size());
+ Cnt = 0;
+ for (const std::pair<int64_t, int> &Pair : Offsets) {
+ SortedIndices[Cnt] = Pair.second;
+ ++Cnt;
+ }
+ return Stride;
+}
+
+// Suppose we are given pointers of the form: %b + x * %s + y * %c
+// where %c is constant. Check if the pointers can be rearranged as follows:
+// %b + 0 * %s + 0
+// %b + 0 * %s + 1
+// %b + 0 * %s + 2
+// ...
+// %b + 0 * %s + w
+//
+// %b + 1 * %s + 0
+// %b + 1 * %s + 1
+// %b + 1 * %s + 2
+// ...
+// %b + 1 * %s + w
+// ...
+//
+// If the pointers can be rearanged in the above pattern, it means that the
+// memory can be accessed with a strided loads of width `w` and stride `%s`.
+bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
+ Type *ElemTy, Align CommonAlignment,
+ SmallVectorImpl<unsigned> &SortedIndices,
+ StridedPtrInfo *SPtrInfo) const {
+ // Group the pointers by constant offset.
+ DenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
+ OffsetToPointerOpIdxMap;
+ for (auto [Idx, Ptr] : enumerate(PointerOps)) {
+ const SCEV *PtrSCEV = SE->getSCEV(Ptr);
+ if (!PtrSCEV)
+ return false;
+
+ const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(PtrSCEV);
+ int64_t Offset = 0;
+ if (Add) {
+ for (int i = 0; i < (int)Add->getNumOperands(); ++i) {
+ const SCEVConstant *SC = dyn_cast<SCEVConstant>(Add->getOperand(i));
+ if (!SC)
+ continue;
+ Offset = (int64_t)(SC->getAPInt().getLimitedValue());
+ break;
+ }
}
+ OffsetToPointerOpIdxMap[Offset].first.push_back(Ptr);
+ OffsetToPointerOpIdxMap[Offset].second.push_back(Idx);
}
- if (!Inst)
- return nullptr;
- SCEVExpander Expander(SE, DL, "strided-load-vec");
- return Expander.expandCodeFor(Stride, Stride->getType(), Inst);
+ int NumOffsets = OffsetToPointerOpIdxMap.size();
+
+ const unsigned Sz = PointerOps.size();
+ unsigned VecSz = Sz;
+ Type *ScalarTy = ElemTy;
+ if (NumOffsets > 1) {
+ if (Sz % NumOffsets != 0)
+ return false;
+ VecSz = Sz / NumOffsets;
+ ScalarTy = Type::getIntNTy(SE->getContext(),
+ DL->getTypeSizeInBits(ElemTy).getFixedValue() *
+ NumOffsets);
+ }
+ FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, VecSz);
+ if (!TTI->isTypeLegal(StridedLoadTy) ||
+ !TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))
+ return false;
+
+ SmallVector<int> SortedOffsetsV;
+ for (auto [K, V] : OffsetToPointerOpIdxMap) {
+ SortedOffsetsV.push_back(K);
+ }
+ llvm::sort(SortedOffsetsV);
+ if (NumOffsets > 1) {
+ int CommonDiff = SortedOffsetsV[1] - SortedOffsetsV[0];
+ if ((CommonDiff) != 1)
+ return false;
+ for (int i = 1; i < (int)SortedOffsetsV.size() - 1; ++i) {
+ if (SortedOffsetsV[i + 1] - SortedOffsetsV[i] != CommonDiff)
+ return false;
+ }
+ }
+
+ int64_t LowestOffset = SortedOffsetsV[0];
+ SmallVector<Value *> &PointerOps0 =
+ OffsetToPointerOpIdxMap[LowestOffset].first;
+ SmallVector<unsigned> &IndicesInAllPointerOps0 =
+ OffsetToPointerOpIdxMap[LowestOffset].second;
+
+ SmallVector<int64_t> Coeffs0;
+ SmallVector<unsigned> SortedIndicesForOffset0;
+ const SCEV *Stride0 = calculateRtStride(PointerOps0, ElemTy, *DL, *SE,
+ SortedIndicesForOffset0, Coeffs0);
+ if (!Stride0)
+ return false;
+ unsigned NumCoeffs0 = Coeffs0.size();
+ if (NumCoeffs0 * NumOffsets != Sz)
+ return false;
+ llvm::sort(Coeffs0);
+
+ SmallVector<unsigned> SortedIndicesDraft;
+ SortedIndicesDraft.resize(Sz);
+ auto updateSortedIndices =
+ [&](SmallVectorImpl<unsigned> &SortedIndicesForOffset,
+ SmallVectorImpl<unsigned> &IndicesInAllPointerOps,
+ int64_t OffsetNum) {
+ unsigned Num = 0;
+ for (unsigned Idx : SortedIndicesForOffset) {
+ SortedIndicesDraft[Num * NumOffsets + OffsetNum] =
+ IndicesInAllPointerOps[Idx];
+ ++Num;
+ }
+ };
+
+ updateSortedIndices(SortedIndicesForOffset0, IndicesInAllPointerOps0, 0);
+
+ SmallVector<int64_t> Coeffs;
+ SmallVector<unsigned> SortedIndicesForOffset;
+ for (int i = 1; i < NumOffsets; ++i) {
+ Coeffs.clear();
+ SortedIndicesForOffset.clear();
+
+ int64_t Offset = SortedOffsetsV[i];
+ SmallVector<Value *> &PointerOpsForOffset =
+ OffsetToPointerOpIdxMap[Offset].first;
+ SmallVector<unsigned> &IndicesInAllPointerOps =
+ OffsetToPointerOpIdxMap[Offset].second;
+ const SCEV *StrideWithinGroup = calculateRtStride(
+ PointerOpsForOffset, ElemTy, *DL, *SE, SortedIndicesForOffset, Coeffs);
+
+ if ((!StrideWithinGroup) || StrideWithinGroup != Stride0) {
+ return false;
+ }
+ if (Coeffs.size() != NumCoeffs0)
+ return false;
+ llvm::sort(Coeffs);
+ for (unsigned i = 0; i < NumCoeffs0; ++i) {
+ if (Coeffs[i] != Coeffs0[i])
+ return false;
+ }
+
+ updateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, i);
+ }
+
+ SortedIndices.clear();
+ SortedIndices = SortedIndicesDraft;
+ if (SPtrInfo) {
+ SPtrInfo->StrideSCEV = Stride0;
+ SPtrInfo->Ty = StridedLoadTy;
+ }
+ return true;
}
static std::pair<InstructionCost, InstructionCost>
@@ -6761,77 +6924,133 @@ isMaskedLoadCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
CompressMask, LoadVecTy);
}
-/// Checks if strided loads can be generated out of \p VL loads with pointers \p
-/// PointerOps:
-/// 1. Target with strided load support is detected.
-/// 2. The number of loads is greater than MinProfitableStridedLoads, or the
-/// potential stride <= MaxProfitableLoadStride and the potential stride is
-/// power-of-2 (to avoid perf regressions for the very small number of loads)
-/// and max distance > number of loads, or potential stride is -1.
-/// 3. The loads are ordered, or number of unordered loads <=
-/// MaxProfitableUnorderedLoads, or loads are in reversed order. (this check is
-/// to avoid extra costs for very expensive shuffles).
-/// 4. Any pointer operand is an instruction with the users outside of the
-/// current graph (for masked gathers extra extractelement instructions
-/// might be required).
-static bool isStridedLoad(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
- ArrayRef<unsigned> Order,
- const TargetTransformInfo &TTI, const DataLayout &DL,
- ScalarEvolution &SE,
- const bool IsAnyPointerUsedOutGraph,
- const int64_t Diff) {
- const size_t Sz = VL.size();
- const uint64_t AbsoluteDiff = std::abs(Diff);
- Type *ScalarTy = VL.front()->getType();
- auto *VecTy = getWidenedType(ScalarTy, Sz);
- if (IsAnyPointerUsedOutGraph ||
- (AbsoluteDiff > Sz &&
- (Sz > MinProfitableStridedLoads ||
- (AbsoluteDiff <= MaxProfitableLoadStride * Sz &&
- AbsoluteDiff % Sz == 0 && has_single_bit(AbsoluteDiff / Sz)))) ||
- Diff == -(static_cast<int64_t>(Sz) - 1)) {
- int64_t Stride = Diff / static_cast<int64_t>(Sz - 1);
- if (Diff != Stride * static_cast<int64_t>(Sz - 1))
+// Same as analyzeRtStrideCandidate, but for constant strides.
+bool BoUpSLP::analyzeConstantStrideCandidate(
+ ArrayRef<Value *> PointerOps, Type *ElemTy, Align CommonAlignment,
+ SmallVectorImpl<unsigned> &SortedIndices, StridedPtrInfo *SPtrInfo,
+ int64_t Diff, Value *Ptr0, Value *PtrN) const {
+ const unsigned Sz = PointerOps.size();
+ SmallVector<int64_t> SortedOffsetsFromBase;
+ SortedOffsetsFromBase.resize(Sz);
+ for (unsigned i = 0; i < Sz; ++i) {
+ Value *Ptr =
+ SortedIndices.empty() ? PointerOps[i] : PointerOps[SortedIndices[i]];
+ SortedOffsetsFromBase[i] =
+ *getPointersDiff(ElemTy, Ptr0, ElemTy, Ptr, *DL, *SE);
+ }
+
+ // Find where the first group ends.
+ assert(SortedOffsetsFromBase.size() > 1);
+ int64_t StrideWithinGroup =
+ SortedOffsetsFromBase[1] - SortedOffsetsFromBase[0];
+ unsigned GroupSize = 1;
+ for (; GroupSize != SortedOffsetsFromBase.size(); ++GroupSize) {
+ if (SortedOffsetsFromBase[GroupSize] -
+ SortedOffsetsFromBase[GroupSize - 1] !=
+ StrideWithinGroup)
+ break;
+ }
+ unsigned VecSz = Sz;
+ Type *ScalarTy = ElemTy;
+ int64_t StrideIntVal = StrideWithinGroup;
+ FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, VecSz);
+
+ if (Sz != GroupSize) {
+ if (Sz % GroupSize != 0)
return false;
- Align Alignment =
- cast<LoadInst>(Order.empty() ? VL.front() : VL[Order.front()])
- ->getAlign();
- if (!TTI.isLegalStridedLoadStore(VecTy, Alignment))
+ VecSz = Sz / GroupSize;
+
+ if (StrideWithinGroup != 1)
return false;
- Value *Ptr0;
- Value *PtrN;
- if (Order.empty()) {
- Ptr0 = PointerOps.front();
- PtrN = PointerOps.back();
- } else {
- Ptr0 = PointerOps[Order.front()];
- PtrN = PointerOps[Order.back()];
- }
- // Iterate through all pointers and check if all distances are
- // unique multiple of Dist.
- SmallSet<int64_t, 4> Dists;
- for (Value *Ptr : PointerOps) {
- int64_t Dist = 0;
- if (Ptr == PtrN)
- Dist = Diff;
- else if (Ptr != Ptr0)
- Dist = *getPointersDiff(ScalarTy, Ptr0, ScalarTy, Ptr, DL, SE);
- // If the strides are not the same or repeated, we can't
- // vectorize.
- if (((Dist / Stride) * Stride) != Dist || !Dists.insert(Dist).second)
+ unsigned VecSz = Sz / GroupSize;
+ ScalarTy = Type::getIntNTy(SE->getContext(),
+ DL->getTypeSizeInBits(ElemTy).getFixedValue() *
+ GroupSize);
+ StridedLoadTy = getWidenedType(ScalarTy, VecSz);
+ if (!TTI->isTypeLegal(StridedLoadTy) ||
+ !TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))
+ return false;
+
+ unsigned PrevGroupStartIdx = 0;
+ unsigned CurrentGroupStartIdx = GroupSize;
+ int64_t StrideBetweenGroups =
+ SortedOffsetsFromBase[GroupSize] - SortedOffsetsFromBase[0];
+ StrideIntVal = StrideBetweenGroups;
+ while (CurrentGroupStartIdx != Sz) {
+ if (SortedOffsetsFromBase[CurrentGroupStartIdx] -
+ SortedOffsetsFromBase[PrevGroupStartIdx] !=
+ StrideBetweenGroups)
break;
+ PrevGroupStartIdx = CurrentGroupStartIdx;
+ CurrentGroupStartIdx += GroupSize;
+ }
+ if (CurrentGroupStartIdx != Sz)
+ return false;
+
+ auto checkGroup = [&](unsigned StartIdx, unsigned GroupSize0,
+ int64_t StrideWithinGroup) -> bool {
+ unsigned GroupEndIdx = StartIdx + 1;
+ for (; GroupEndIdx != Sz; ++GroupEndIdx) {
+ if (SortedOffsetsFromBase[GroupEndIdx] -
+ SortedOffsetsFromBase[GroupEndIdx - 1] !=
+ StrideWithinGroup)
+ break;
+ }
+ return (GroupEndIdx - StartIdx == GroupSize0);
+ };
+ for (unsigned i = 0; i < Sz; i += GroupSize) {
+ if (!checkGroup(i, GroupSize, StrideWithinGroup))
+ return false;
}
- if (Dists.size() == Sz)
- return true;
+ }
+
+ // Try to generate strided load node if:
+ // 1. Target with strided load support is detected.
+ // 2. The number of loads is greater than MinProfitableStridedLoads,
+ // or the potential stride <= MaxProfitableLoadStride and the
+ // potential stride is power-of-2 (to avoid perf regressions for the very
+ // small number of loads) and max distance > number of loads, or potential
+ // stride is -1.
+ // 3. The loads are ordered, or number of unordered loads <=
+ // MaxProfitableUnorderedLoads, or loads are in reversed order.
+ // (this check is to avoid extra costs for very expensive shuffles).
+ // 4. Any pointer operand is an instruction with the users outside of the
+ // current graph (for masked gathers extra extractelement instructions
+ // might be required).
+
+ if (!TTI->isTypeLegal(StridedLoadTy) ||
+ !TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))
+ return false;
+
+ // Simple check if not a strided access - clear order.
+ bool IsPossibleStrided = Diff % (VecSz - 1) == 0;
+ auto IsAnyPointerUsedOutGraph =
+ IsPossibleStrided && any_of(PointerOps, [&](Value *V) {
+ return isa<Instruction>(V) && any_of(V->users(), [&](User *U) {
+ return !isVectorized(U) && !MustGather.contains(U);
+ });
+ });
+ const unsigned AbsoluteDiff = std::abs(Diff);
+ if (IsAnyPointerUsedOutGraph ||
+ ((VecSz > MinProfitableStridedLoads ||
+ (AbsoluteDiff <= MaxProfitableLoadStride * VecSz &&
+ has_single_bit(AbsoluteDiff))) &&
+ AbsoluteDiff > VecSz) ||
+ Diff == -(static_cast<int>(VecSz) - 1)) {
+ if (SPtrInfo) {
+ Type *StrideTy = DL->getIndexType(Ptr0->getType());
+ SPtrInfo->StrideVal = ConstantInt::get(StrideTy, StrideIntVal);
+ SPtrInfo->Ty = StridedLoadTy;
+ }
+ return true;
}
return false;
}
-BoUpSLP::LoadsState
-BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
- SmallVectorImpl<unsigned> &Order,
- SmallVectorImpl<Value *> &PointerOps,
- unsigned *BestVF, bool TryRecursiveCheck) const {
+BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
+ ArrayRef<Value *> VL, const Value *VL0, SmallVectorImpl<unsigned> &Order,
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo *SPtrInfo,
+ unsigned *BestVF, bool TryRecursiveCheck) const {
// Check that a vectorized load would load the same memory as a scalar
// load. For example, we don't want to vectorize loads that are smaller
// than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
@@ -6868,11 +7087,10 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
auto *VecTy = getWidenedType(ScalarTy, Sz);
Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
if (!IsSorted) {
- if (Sz > MinProfitableStridedLoads && TTI->isTypeLegal(VecTy)) {
- if (TTI->isLegalStridedLoadStore(VecTy, CommonAlignment) &&
- calculateRtStride(PointerOps, ScalarTy, *DL, *SE, Order))
- return LoadsState::StridedVectorize;
- }
+ if (Sz > MinProfitableStridedLoads &&
+ analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order,
+ SPtrInfo))
+ return LoadsState::StridedVectorize;
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment))
@@ -6905,17 +7123,8 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
}))
return LoadsState::CompressVectorize;
// Simple check if not a strided access - clear order.
- bool IsPossibleStrided = *Diff % (Sz - 1) == 0;
- // Try to generate strided load node.
- auto IsAnyPointerUsedOutGraph =
- IsPossibleStrided && any_of(PointerOps, [&](Value *V) {
- return isa<Instruction>(V) && any_of(V->users(), [&](User *U) {
- return !isVectorized(U) && !MustGather.contains(U);
- });
- });
- if (IsPossibleStrided &&
- isStridedLoad(VL, PointerOps, Order, *TTI, *DL, *SE,
- IsAnyPointerUsedOutGraph, *Diff))
+ if (analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
+ Order, SPtrInfo, *Diff, Ptr0, PtrN))
return LoadsState::StridedVectorize;
}
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
@@ -6999,9 +7208,9 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
SmallVector<unsigned> Order;
SmallVector<Value *> PointerOps;
- LoadsState LS =
- canVectorizeLoads(Slice, Slice.front(), Order, PointerOps, BestVF,
- /*TryRecursiveCheck=*/false);
+ LoadsState LS = canVectorizeLoads(Slice, Slice.front(), Order,
+ PointerOps, SPtrInfo, BestVF,
+ /*TryRecursiveCheck=*/false);
// Check that the sorted loads are consecutive.
if (LS == LoadsState::Gather) {
if (BestVF) {
@@ -9179,8 +9388,9 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
// Try to build vector load.
ArrayRef<Value *> Values(
reinterpret_cast<Value *const *>(Slice.begin()), Slice.size());
- LoadsState LS = canVectorizeLoads(Values, Slice.front(), CurrentOrder,
- PointerOps, &BestVF);
+ LoadsState LS =
+ canVectorizeLoads(Values, Slice.front(), CurrentOrder, PointerOps,
+ /*SPtrInfo =*/nullptr, &BestVF);
if (LS != LoadsState::Gather ||
(BestVF > 1 && static_cast<unsigned>(NumElts) == 2 * BestVF)) {
if (LS == LoadsState::ScatterVectorize) {
@@ -9785,7 +9995,7 @@ getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
- SmallVectorImpl<Value *> &PointerOps) {
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo *SPtrInfo) {
assert(S.getMainOp() &&
"Expected instructions with same/alternate opcodes only.");
@@ -9887,7 +10097,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
});
});
};
- switch (canVectorizeLoads(VL, VL0, CurrentOrder, PointerOps)) {
+ switch (canVectorizeLoads(VL, VL0, CurrentOrder, PointerOps, SPtrInfo)) {
case LoadsState::Vectorize:
return TreeEntry::Vectorize;
case LoadsState::CompressVectorize:
@@ -11319,8 +11529,9 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
OrdersType CurrentOrder;
SmallVector<Value *> PointerOps;
+ StridedPtrInfo SPtrInfo;
TreeEntry::EntryState State = getScalarsVectorizationState(
- S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
+ S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps, &SPtrInfo);
if (State == TreeEntry::NeedToGather) {
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
@@ -11480,6 +11691,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Vectorizing non-consecutive loads with `llvm.masked.gather`.
TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
+ TreeEntryToStridedPtrInfoMap[TE] = SPtrInfo;
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (strided LoadInst).\n";
TE->dump());
break;
@@ -12975,10 +13187,20 @@ void BoUpSLP::transformNodes() {
InstructionCost StridedCost = TTI->getStridedMemoryOpCost(
Instruction::Load, VecTy, BaseLI->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment, CostKind, BaseLI);
- if (StridedCost < OriginalVecCost)
+ if (StridedCost < OriginalVecCost) {
// Strided load is more profitable than consecutive load + reverse -
// transform the node to strided load.
+ // TODO: StrideTy =
+ // DL->getIndexType(E.scalars.front()->getPointerOperand()->getType());
+ StridedPtrInfo SPtrInfo;
+ Type *StrideTy = Type::getIntNTy(
+ SE->getContext(),
+ DL->getTypeStoreSizeInBits(E.Scalars.front()->getType()));
+ SPtrInfo.StrideVal = ConstantInt::get(StrideTy, 1);
+ SPtrInfo.Ty = VecTy;
+ TreeEntryToStridedPtrInfoMap[&E] = SPtrInfo;
E.State = TreeEntry::StridedVectorize;
+ }
}
break;
}
@@ -19436,6 +19658,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
LoadInst *LI = cast<LoadInst>(VL0);
Instruction *NewLI;
+ FixedVectorType *StridedLoadTy = nullptr;
Value *PO = LI->getPointerOperand();
if (E->State == TreeEntry::Vectorize) {
NewLI = Builder.CreateAlignedLoad(VecTy, PO, LI->getAlign());
@@ -19473,40 +19696,37 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Value *Ptr0 = cast<LoadInst>(E->Scalars.front())->getPointerOperand();
Value *PtrN = cast<LoadInst>(E->Scalars.back())->getPointerOperand();
PO = IsReverseOrder ? PtrN : Ptr0;
- std::optional<int64_t> Diff = getPointersDiff(
- VL0->getType(), Ptr0, VL0->getType(), PtrN, *DL, *SE);
Type *StrideTy = DL->getIndexType(PO->getType());
Value *StrideVal;
- if (Diff) {
- int64_t Stride =
- *Diff / (static_cast<int64_t>(E->Scalars.size()) - 1);
- StrideVal =
- ConstantInt::get(StrideTy, (IsReverseOrder ? -1 : 1) * Stride *
- DL->getTypeAllocSize(ScalarTy));
- } else {
- SmallVector<Value *> PointerOps(E->Scalars.size(), nullptr);
- transform(E->Scalars, PointerOps.begin(), [](Value *V) {
- return cast<LoadInst>(V)->getPointerOperand();
- });
- OrdersType Order;
- std::optional<Value *> Stride =
- calculateRtStride(PointerOps, ScalarTy, *DL, *SE, Order,
- &*Builder.GetInsertPoint());
- Value *NewStride =
- Builder.CreateIntCast(*Stride, StrideTy, /*isSigned=*/true);
- StrideVal = Builder.CreateMul(
- NewStride,
- ConstantInt::get(
- StrideTy,
- (IsReverseOrder ? -1 : 1) *
- static_cast<int>(DL->getTypeAllocSize(ScalarTy))));
- }
+
+ StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap[E];
+ StridedLoadTy = SPtrInfo.Ty;
+ assert(StridedLoadTy);
+ unsigned StridedLoadEC =
+ StridedLoadTy->getElementCount().getKnownMinValue();
+
+ Value *Stride = SPtrInfo.StrideVal;
+ if (!Stride) {
+ const SCEV *StrideSCEV = SPtrInfo.StrideSCEV;
+ assert(StrideSCEV);
+ SCEVExpander Expander(*SE, *DL, "strided-load-vec");
+ Stride = Expander.expandCodeFor(StrideSCEV, StrideSCEV->getType(),
+ &*Builder.GetInsertPoint());
+ }
+ Value *NewStride =
+ Builder.CreateIntCast(Stride, StrideTy, /*isSigned=*/true);
+ StrideVal = Builder.CreateMul(
+ NewStride, ConstantInt::get(
+ StrideTy, (IsReverseOrder ? -1 : 1) *
+ static_cast<int>(
+ DL->getTypeAllocSize(ScalarTy))));
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
auto *Inst = Builder.CreateIntrinsic(
Intrinsic::experimental_vp_strided_load,
- {VecTy, PO->getType(), StrideTy},
- {PO, StrideVal, Builder.getAllOnesMask(VecTy->getElementCount()),
- Builder.getInt32(E->Scalars.size())});
+ {StridedLoadTy, PO->getType(), StrideTy},
+ {PO, StrideVal,
+ Builder.getAllOnesMask(ElementCount::getFixed(StridedLoadEC)),
+ Builder.getInt32(StridedLoadEC)});
Inst->addParamAttr(
/*ArgNo=*/0,
Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
@@ -19543,6 +19763,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
? NewLI
: ::propagateMetadata(NewLI, E->Scalars);
+ if (StridedLoadTy)
+ V = Builder.CreateBitOrPointerCast(V, VecTy);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
new file mode 100644
index 0000000000000..38cf1214081fa
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
@@ -0,0 +1,483 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=riscv64 -mattr=+m,+v,+unaligned-vector-mem \
+; RUN: -riscv-v-slp-prefer-alt-opc-vectorization=true \
+; RUN: -passes=slp-vectorizer -S < %s | FileCheck %s
+; Function Attrs: nounwind uwtable vscale_range(8,1024)
+define i32 @x264_pixel_satd_8x4(ptr %pix1, i32 %i_pix1, ptr %pix2, i32 %i_pix2) {
+; CHECK-LABEL: define i32 @x264_pixel_satd_8x4(
+; CHECK-SAME: ptr [[PIX1:%.*]], i32 [[I_PIX1:%.*]], ptr [[PIX2:%.*]], i32 [[I_PIX2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[I_PIX1]] to i64
+; CHECK-NEXT: [[IDX_EXT63:%.*]] = sext i32 [[I_PIX2]] to i64
+; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[PIX1]], i64 4
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[PIX2]], i64 4
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[IDX_EXT]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[PIX1]], i64 [[TMP0]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast <4 x i32> [[TMP1]] to <16 x i8>
+; CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[TMP19]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[IDX_EXT63]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[PIX2]], i64 [[TMP4]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP28:%.*]] = bitcast <4 x i32> [[TMP5]] to <16 x i8>
+; CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[TMP28]] to <16 x i32>
+; CHECK-NEXT: [[TMP30:%.*]] = sub nsw <16 x i32> [[TMP20]], [[TMP29]]
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[IDX_EXT]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[ARRAYIDX3]], i64 [[TMP9]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast <4 x i32> [[TMP10]] to <16 x i8>
+; CHECK-NEXT: [[TMP39:%.*]] = zext <16 x i8> [[TMP38]] to <16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[IDX_EXT63]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[ARRAYIDX5]], i64 [[TMP13]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP47:%.*]] = bitcast <4 x i32> [[TMP14]] to <16 x i8>
+; CHECK-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[TMP47]] to <16 x i32>
+; CHECK-NEXT: [[TMP49:%.*]] = sub nsw <16 x i32> [[TMP39]], [[TMP48]]
+; CHECK-NEXT: [[TMP50:%.*]] = shl nsw <16 x i32> [[TMP49]], splat (i32 16)
+; CHECK-NEXT: [[TMP51:%.*]] = add nsw <16 x i32> [[TMP50]], [[TMP30]]
+; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT: [[TMP53:%.*]] = add nsw <16 x i32> [[TMP52]], [[TMP51]]
+; CHECK-NEXT: [[TMP54:%.*]] = sub nsw <16 x i32> [[TMP52]], [[TMP51]]
+; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <16 x i32> [[TMP53]], <16 x i32> [[TMP54]], <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <16 x i32> [[TMP55]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
+; CHECK-NEXT: [[TMP57:%.*]] = add nsw <16 x i32> [[TMP55]], [[TMP56]]
+; CHECK-NEXT: [[TMP58:%.*]] = sub nsw <16 x i32> [[TMP55]], [[TMP56]]
+; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP57]], <16 x i32> [[TMP58]], <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 20, i32 21, i32 6, i32 7, i32 24, i32 25, i32 10, i32 11, i32 28, i32 29, i32 14, i32 15>
+; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP61:%.*]] = sub nsw <16 x i32> [[TMP59]], [[TMP60]]
+; CHECK-NEXT: [[TMP62:%.*]] = add nsw <16 x i32> [[TMP59]], [[TMP60]]
+; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP61]], <16 x i32> [[TMP62]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP64:%.*]] = shufflevector <16 x i32> [[TMP63]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP65:%.*]] = add nsw <16 x i32> [[TMP63]], [[TMP64]]
+; CHECK-NEXT: [[TMP66:%.*]] = sub nsw <16 x i32> [[TMP63]], [[TMP64]]
+; CHECK-NEXT: [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP65]], <16 x i32> [[TMP66]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP68:%.*]] = lshr <16 x i32> [[TMP67]], splat (i32 15)
+; CHECK-NEXT: [[TMP69:%.*]] = and <16 x i32> [[TMP68]], splat (i32 65537)
+; CHECK-NEXT: [[TMP70:%.*]] = mul nuw <16 x i32> [[TMP69]], splat (i32 65535)
+; CHECK-NEXT: [[TMP71:%.*]] = add <16 x i32> [[TMP70]], [[TMP67]]
+; CHECK-NEXT: [[TMP72:%.*]] = xor <16 x i32> [[TMP71]], [[TMP70]]
+; CHECK-NEXT: [[TMP73:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP72]])
+; CHECK-NEXT: [[CONV118:%.*]] = and i32 [[TMP73]], 65535
+; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[TMP73]], 16
+; CHECK-NEXT: [[ADD119:%.*]] = add nuw nsw i32 [[CONV118]], [[SHR]]
+; CHECK-NEXT: [[SHR120:%.*]] = lshr i32 [[ADD119]], 1
+; CHECK-NEXT: ret i32 [[SHR120]]
+;
+entry:
+ %idx.ext = sext i32 %i_pix1 to i64
+ %idx.ext63 = sext i32 %i_pix2 to i64
+ %0 = load i8, ptr %pix1, align 1
+ %conv = zext i8 %0 to i32
+ %1 = load i8, ptr %pix2, align 1
+ %conv2 = zext i8 %1 to i32
+ %sub = sub nsw i32 %conv, %conv2
+ %arrayidx3 = getelementptr inbounds nuw i8, ptr %pix1, i64 4
+ %2 = load i8, ptr %arrayidx3, align 1
+ %conv4 = zext i8 %2 to i32
+ %arrayidx5 = getelementptr inbounds nuw i8, ptr %pix2, i64 4
+ %3 = load i8, ptr %arrayidx5, align 1
+ %conv6 = zext i8 %3 to i32
+ %sub7 = sub nsw i32 %conv4, %conv6
+ %shl = shl nsw i32 %sub7, 16
+ %add = add nsw i32 %shl, %sub
+ %arrayidx8 = getelementptr inbounds nuw i8, ptr %pix1, i64 1
+ %4 = load i8, ptr %arrayidx8, align 1
+ %conv9 = zext i8 %4 to i32
+ %arrayidx10 = getelementptr inbounds nuw i8, ptr %pix2, i64 1
+ %5 = load i8, ptr %arrayidx10, align 1
+ %conv11 = zext i8 %5 to i32
+ %sub12 = sub nsw i32 %conv9, %conv11
+ %arrayidx13 = getelementptr inbounds nuw i8, ptr %pix1, i64 5
+ %6 = load i8, ptr %arrayidx13, align 1
+ %conv14 = zext i8 %6 to i32
+ %arrayidx15 = getelementptr inbounds nuw i8, ptr %pix2, i64 5
+ %7 = load i8, ptr %arrayidx15, align 1
+ %conv16 = zext i8 %7 to i32
+ %sub17 = sub nsw i32 %conv14, %conv16
+ %shl18 = shl nsw i32 %sub17, 16
+ %add19 = add nsw i32 %shl18, %sub12
+ %arrayidx20 = getelementptr inbounds nuw i8, ptr %pix1, i64 2
+ %8 = load i8, ptr %arrayidx20, align 1
+ %conv21 = zext i8 %8 to i32
+ %arrayidx22 = getelementptr inbounds nuw i8, ptr %pix2, i64 2
+ %9 = load i8, ptr %arrayidx22, align 1
+ %conv23 = zext i8 %9 to i32
+ %sub24 = sub nsw i32 %conv21, %conv23
+ %arrayidx25 = getelementptr inbounds nuw i8, ptr %pix1, i64 6
+ %10 = load i8, ptr %arrayidx25, align 1
+ %conv26 = zext i8 %10 to i32
+ %arrayidx27 = getelementptr inbounds nuw i8, ptr %pix2, i64 6
+ %11 = load i8, ptr %arrayidx27, align 1
+ %conv28 = zext i8 %11 to i32
+ %sub29 = sub nsw i32 %conv26, %conv28
+ %shl30 = shl nsw i32 %sub29, 16
+ %add31 = add nsw i32 %shl30, %sub24
+ %arrayidx32 = getelementptr inbounds nuw i8, ptr %pix1, i64 3
+ %12 = load i8, ptr %arrayidx32, align 1
+ %conv33 = zext i8 %12 to i32
+ %arrayidx34 = getelementptr inbounds nuw i8, ptr %pix2, i64 3
+ %13 = load i8, ptr %arrayidx34, align 1
+ %conv35 = zext i8 %13 to i32
+ %sub36 = sub nsw i32 %conv33, %conv35
+ %arrayidx37 = getelementptr inbounds nuw i8, ptr %pix1, i64 7
+ %14 = load i8, ptr %arrayidx37, align 1
+ %conv38 = zext i8 %14 to i32
+ %arrayidx39 = getelementptr inbounds nuw i8, ptr %pix2, i64 7
+ %15 = load i8, ptr %arrayidx39, align 1
+ %conv40 = zext i8 %15 to i32
+ %sub41 = sub nsw i32 %conv38, %conv40
+ %shl42 = shl nsw i32 %sub41, 16
+ %add43 = add nsw i32 %shl42, %sub36
+ %add44 = add nsw i32 %add19, %add
+ %sub45 = sub nsw i32 %add, %add19
+ %add46 = add nsw i32 %add43, %add31
+ %sub47 = sub nsw i32 %add31, %add43
+ %add48 = add nsw i32 %add46, %add44
+ %sub51 = sub nsw i32 %add44, %add46
+ %add55 = add nsw i32 %sub47, %sub45
+ %sub59 = sub nsw i32 %sub45, %sub47
+ %add.ptr = getelementptr inbounds i8, ptr %pix1, i64 %idx.ext
+ %add.ptr64 = getelementptr inbounds i8, ptr %pix2, i64 %idx.ext63
+ %16 = load i8, ptr %add.ptr, align 1
+ %conv.1 = zext i8 %16 to i32
+ %17 = load i8, ptr %add.ptr64, align 1
+ %conv2.1 = zext i8 %17 to i32
+ %sub.1 = sub nsw i32 %conv.1, %conv2.1
+ %arrayidx3.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 4
+ %18 = load i8, ptr %arrayidx3.1, align 1
+ %conv4.1 = zext i8 %18 to i32
+ %arrayidx5.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 4
+ %19 = load i8, ptr %arrayidx5.1, align 1
+ %conv6.1 = zext i8 %19 to i32
+ %sub7.1 = sub nsw i32 %conv4.1, %conv6.1
+ %shl.1 = shl nsw i32 %sub7.1, 16
+ %add.1 = add nsw i32 %shl.1, %sub.1
+ %arrayidx8.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 1
+ %20 = load i8, ptr %arrayidx8.1, align 1
+ %conv9.1 = zext i8 %20 to i32
+ %arrayidx10.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 1
+ %21 = load i8, ptr %arrayidx10.1, align 1
+ %conv11.1 = zext i8 %21 to i32
+ %sub12.1 = sub nsw i32 %conv9.1, %conv11.1
+ %arrayidx13.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 5
+ %22 = load i8, ptr %arrayidx13.1, align 1
+ %conv14.1 = zext i8 %22 to i32
+ %arrayidx15.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 5
+ %23 = load i8, ptr %arrayidx15.1, align 1
+ %conv16.1 = zext i8 %23 to i32
+ %sub17.1 = sub nsw i32 %conv14.1, %conv16.1
+ %shl18.1 = shl nsw i32 %sub17.1, 16
+ %add19.1 = add nsw i32 %shl18.1, %sub12.1
+ %arrayidx20.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 2
+ %24 = load i8, ptr %arrayidx20.1, align 1
+ %conv21.1 = zext i8 %24 to i32
+ %arrayidx22.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 2
+ %25 = load i8, ptr %arrayidx22.1, align 1
+ %conv23.1 = zext i8 %25 to i32
+ %sub24.1 = sub nsw i32 %conv21.1, %conv23.1
+ %arrayidx25.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 6
+ %26 = load i8, ptr %arrayidx25.1, align 1
+ %conv26.1 = zext i8 %26 to i32
+ %arrayidx27.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 6
+ %27 = load i8, ptr %arrayidx27.1, align 1
+ %conv28.1 = zext i8 %27 to i32
+ %sub29.1 = sub nsw i32 %conv26.1, %conv28.1
+ %shl30.1 = shl nsw i32 %sub29.1, 16
+ %add31.1 = add nsw i32 %shl30.1, %sub24.1
+ %arrayidx32.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 3
+ %28 = load i8, ptr %arrayidx32.1, align 1
+ %conv33.1 = zext i8 %28 to i32
+ %arrayidx34.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 3
+ %29 = load i8, ptr %arrayidx34.1, align 1
+ %conv35.1 = zext i8 %29 to i32
+ %sub36.1 = sub nsw i32 %conv33.1, %conv35.1
+ %arrayidx37.1 = getelementptr inbounds nuw i8, ptr %add.ptr, i64 7
+ %30 = load i8, ptr %arrayidx37.1, align 1
+ %conv38.1 = zext i8 %30 to i32
+ %arrayidx39.1 = getelementptr inbounds nuw i8, ptr %add.ptr64, i64 7
+ %31 = load i8, ptr %arrayidx39.1, align 1
+ %conv40.1 = zext i8 %31 to i32
+ %sub41.1 = sub nsw i32 %conv38.1, %conv40.1
+ %shl42.1 = shl nsw i32 %sub41.1, 16
+ %add43.1 = add nsw i32 %shl42.1, %sub36.1
+ %add44.1 = add nsw i32 %add19.1, %add.1
+ %sub45.1 = sub nsw i32 %add.1, %add19.1
+ %add46.1 = add nsw i32 %add43.1, %add31.1
+ %sub47.1 = sub nsw i32 %add31.1, %add43.1
+ %add48.1 = add nsw i32 %add46.1, %add44.1
+ %sub51.1 = sub nsw i32 %add44.1, %add46.1
+ %add55.1 = add nsw i32 %sub47.1, %sub45.1
+ %sub59.1 = sub nsw i32 %sub45.1, %sub47.1
+ %add.ptr.1 = getelementptr inbounds i8, ptr %add.ptr, i64 %idx.ext
+ %add.ptr64.1 = getelementptr inbounds i8, ptr %add.ptr64, i64 %idx.ext63
+ %32 = load i8, ptr %add.ptr.1, align 1
+ %conv.2 = zext i8 %32 to i32
+ %33 = load i8, ptr %add.ptr64.1, align 1
+ %conv2.2 = zext i8 %33 to i32
+ %sub.2 = sub nsw i32 %conv.2, %conv2.2
+ %arrayidx3.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 4
+ %34 = load i8, ptr %arrayidx3.2, align 1
+ %conv4.2 = zext i8 %34 to i32
+ %arrayidx5.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 4
+ %35 = load i8, ptr %arrayidx5.2, align 1
+ %conv6.2 = zext i8 %35 to i32
+ %sub7.2 = sub nsw i32 %conv4.2, %conv6.2
+ %shl.2 = shl nsw i32 %sub7.2, 16
+ %add.2 = add nsw i32 %shl.2, %sub.2
+ %arrayidx8.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 1
+ %36 = load i8, ptr %arrayidx8.2, align 1
+ %conv9.2 = zext i8 %36 to i32
+ %arrayidx10.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 1
+ %37 = load i8, ptr %arrayidx10.2, align 1
+ %conv11.2 = zext i8 %37 to i32
+ %sub12.2 = sub nsw i32 %conv9.2, %conv11.2
+ %arrayidx13.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 5
+ %38 = load i8, ptr %arrayidx13.2, align 1
+ %conv14.2 = zext i8 %38 to i32
+ %arrayidx15.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 5
+ %39 = load i8, ptr %arrayidx15.2, align 1
+ %conv16.2 = zext i8 %39 to i32
+ %sub17.2 = sub nsw i32 %conv14.2, %conv16.2
+ %shl18.2 = shl nsw i32 %sub17.2, 16
+ %add19.2 = add nsw i32 %shl18.2, %sub12.2
+ %arrayidx20.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 2
+ %40 = load i8, ptr %arrayidx20.2, align 1
+ %conv21.2 = zext i8 %40 to i32
+ %arrayidx22.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 2
+ %41 = load i8, ptr %arrayidx22.2, align 1
+ %conv23.2 = zext i8 %41 to i32
+ %sub24.2 = sub nsw i32 %conv21.2, %conv23.2
+ %arrayidx25.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 6
+ %42 = load i8, ptr %arrayidx25.2, align 1
+ %conv26.2 = zext i8 %42 to i32
+ %arrayidx27.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 6
+ %43 = load i8, ptr %arrayidx27.2, align 1
+ %conv28.2 = zext i8 %43 to i32
+ %sub29.2 = sub nsw i32 %conv26.2, %conv28.2
+ %shl30.2 = shl nsw i32 %sub29.2, 16
+ %add31.2 = add nsw i32 %shl30.2, %sub24.2
+ %arrayidx32.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 3
+ %44 = load i8, ptr %arrayidx32.2, align 1
+ %conv33.2 = zext i8 %44 to i32
+ %arrayidx34.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 3
+ %45 = load i8, ptr %arrayidx34.2, align 1
+ %conv35.2 = zext i8 %45 to i32
+ %sub36.2 = sub nsw i32 %conv33.2, %conv35.2
+ %arrayidx37.2 = getelementptr inbounds nuw i8, ptr %add.ptr.1, i64 7
+ %46 = load i8, ptr %arrayidx37.2, align 1
+ %conv38.2 = zext i8 %46 to i32
+ %arrayidx39.2 = getelementptr inbounds nuw i8, ptr %add.ptr64.1, i64 7
+ %47 = load i8, ptr %arrayidx39.2, align 1
+ %conv40.2 = zext i8 %47 to i32
+ %sub41.2 = sub nsw i32 %conv38.2, %conv40.2
+ %shl42.2 = shl nsw i32 %sub41.2, 16
+ %add43.2 = add nsw i32 %shl42.2, %sub36.2
+ %add44.2 = add nsw i32 %add19.2, %add.2
+ %sub45.2 = sub nsw i32 %add.2, %add19.2
+ %add46.2 = add nsw i32 %add43.2, %add31.2
+ %sub47.2 = sub nsw i32 %add31.2, %add43.2
+ %add48.2 = add nsw i32 %add46.2, %add44.2
+ %sub51.2 = sub nsw i32 %add44.2, %add46.2
+ %add55.2 = add nsw i32 %sub47.2, %sub45.2
+ %sub59.2 = sub nsw i32 %sub45.2, %sub47.2
+ %add.ptr.2 = getelementptr inbounds i8, ptr %add.ptr.1, i64 %idx.ext
+ %add.ptr64.2 = getelementptr inbounds i8, ptr %add.ptr64.1, i64 %idx.ext63
+ %48 = load i8, ptr %add.ptr.2, align 1
+ %conv.3 = zext i8 %48 to i32
+ %49 = load i8, ptr %add.ptr64.2, align 1
+ %conv2.3 = zext i8 %49 to i32
+ %sub.3 = sub nsw i32 %conv.3, %conv2.3
+ %arrayidx3.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 4
+ %50 = load i8, ptr %arrayidx3.3, align 1
+ %conv4.3 = zext i8 %50 to i32
+ %arrayidx5.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 4
+ %51 = load i8, ptr %arrayidx5.3, align 1
+ %conv6.3 = zext i8 %51 to i32
+ %sub7.3 = sub nsw i32 %conv4.3, %conv6.3
+ %shl.3 = shl nsw i32 %sub7.3, 16
+ %add.3 = add nsw i32 %shl.3, %sub.3
+ %arrayidx8.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 1
+ %52 = load i8, ptr %arrayidx8.3, align 1
+ %conv9.3 = zext i8 %52 to i32
+ %arrayidx10.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 1
+ %53 = load i8, ptr %arrayidx10.3, align 1
+ %conv11.3 = zext i8 %53 to i32
+ %sub12.3 = sub nsw i32 %conv9.3, %conv11.3
+ %arrayidx13.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 5
+ %54 = load i8, ptr %arrayidx13.3, align 1
+ %conv14.3 = zext i8 %54 to i32
+ %arrayidx15.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 5
+ %55 = load i8, ptr %arrayidx15.3, align 1
+ %conv16.3 = zext i8 %55 to i32
+ %sub17.3 = sub nsw i32 %conv14.3, %conv16.3
+ %shl18.3 = shl nsw i32 %sub17.3, 16
+ %add19.3 = add nsw i32 %shl18.3, %sub12.3
+ %arrayidx20.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 2
+ %56 = load i8, ptr %arrayidx20.3, align 1
+ %conv21.3 = zext i8 %56 to i32
+ %arrayidx22.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 2
+ %57 = load i8, ptr %arrayidx22.3, align 1
+ %conv23.3 = zext i8 %57 to i32
+ %sub24.3 = sub nsw i32 %conv21.3, %conv23.3
+ %arrayidx25.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 6
+ %58 = load i8, ptr %arrayidx25.3, align 1
+ %conv26.3 = zext i8 %58 to i32
+ %arrayidx27.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 6
+ %59 = load i8, ptr %arrayidx27.3, align 1
+ %conv28.3 = zext i8 %59 to i32
+ %sub29.3 = sub nsw i32 %conv26.3, %conv28.3
+ %shl30.3 = shl nsw i32 %sub29.3, 16
+ %add31.3 = add nsw i32 %shl30.3, %sub24.3
+ %arrayidx32.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 3
+ %60 = load i8, ptr %arrayidx32.3, align 1
+ %conv33.3 = zext i8 %60 to i32
+ %arrayidx34.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 3
+ %61 = load i8, ptr %arrayidx34.3, align 1
+ %conv35.3 = zext i8 %61 to i32
+ %sub36.3 = sub nsw i32 %conv33.3, %conv35.3
+ %arrayidx37.3 = getelementptr inbounds nuw i8, ptr %add.ptr.2, i64 7
+ %62 = load i8, ptr %arrayidx37.3, align 1
+ %conv38.3 = zext i8 %62 to i32
+ %arrayidx39.3 = getelementptr inbounds nuw i8, ptr %add.ptr64.2, i64 7
+ %63 = load i8, ptr %arrayidx39.3, align 1
+ %conv40.3 = zext i8 %63 to i32
+ %sub41.3 = sub nsw i32 %conv38.3, %conv40.3
+ %shl42.3 = shl nsw i32 %sub41.3, 16
+ %add43.3 = add nsw i32 %shl42.3, %sub36.3
+ %add44.3 = add nsw i32 %add19.3, %add.3
+ %sub45.3 = sub nsw i32 %add.3, %add19.3
+ %add46.3 = add nsw i32 %add43.3, %add31.3
+ %sub47.3 = sub nsw i32 %add31.3, %add43.3
+ %add48.3 = add nsw i32 %add46.3, %add44.3
+ %sub51.3 = sub nsw i32 %add44.3, %add46.3
+ %add55.3 = add nsw i32 %sub47.3, %sub45.3
+ %sub59.3 = sub nsw i32 %sub45.3, %sub47.3
+ %add78 = add nsw i32 %add48.1, %add48
+ %sub86 = sub nsw i32 %add48, %add48.1
+ %add94 = add nsw i32 %add48.3, %add48.2
+ %sub102 = sub nsw i32 %add48.2, %add48.3
+ %add103 = add nsw i32 %add94, %add78
+ %sub104 = sub nsw i32 %add78, %add94
+ %add105 = add nsw i32 %sub102, %sub86
+ %sub106 = sub nsw i32 %sub86, %sub102
+ %shr.i = lshr i32 %add103, 15
+ %and.i = and i32 %shr.i, 65537
+ %mul.i = mul nuw i32 %and.i, 65535
+ %add.i = add i32 %mul.i, %add103
+ %xor.i = xor i32 %add.i, %mul.i
+ %shr.i169 = lshr i32 %add105, 15
+ %and.i170 = and i32 %shr.i169, 65537
+ %mul.i171 = mul nuw i32 %and.i170, 65535
+ %add.i172 = add i32 %mul.i171, %add105
+ %xor.i173 = xor i32 %add.i172, %mul.i171
+ %shr.i174 = lshr i32 %sub104, 15
+ %and.i175 = and i32 %shr.i174, 65537
+ %mul.i176 = mul nuw i32 %and.i175, 65535
+ %add.i177 = add i32 %mul.i176, %sub104
+ %xor.i178 = xor i32 %add.i177, %mul.i176
+ %shr.i179 = lshr i32 %sub106, 15
+ %and.i180 = and i32 %shr.i179, 65537
+ %mul.i181 = mul nuw i32 %and.i180, 65535
+ %add.i182 = add i32 %mul.i181, %sub106
+ %xor.i183 = xor i32 %add.i182, %mul.i181
+ %add110 = add i32 %xor.i173, %xor.i
+ %add112 = add i32 %add110, %xor.i178
+ %add113 = add i32 %add112, %xor.i183
+ %add78.1 = add nsw i32 %add55.1, %add55
+ %sub86.1 = sub nsw i32 %add55, %add55.1
+ %add94.1 = add nsw i32 %add55.3, %add55.2
+ %sub102.1 = sub nsw i32 %add55.2, %add55.3
+ %add103.1 = add nsw i32 %add94.1, %add78.1
+ %sub104.1 = sub nsw i32 %add78.1, %add94.1
+ %add105.1 = add nsw i32 %sub102.1, %sub86.1
+ %sub106.1 = sub nsw i32 %sub86.1, %sub102.1
+ %shr.i.1 = lshr i32 %add103.1, 15
+ %and.i.1 = and i32 %shr.i.1, 65537
+ %mul.i.1 = mul nuw i32 %and.i.1, 65535
+ %add.i.1 = add i32 %mul.i.1, %add103.1
+ %xor.i.1 = xor i32 %add.i.1, %mul.i.1
+ %shr.i169.1 = lshr i32 %add105.1, 15
+ %and.i170.1 = and i32 %shr.i169.1, 65537
+ %mul.i171.1 = mul nuw i32 %and.i170.1, 65535
+ %add.i172.1 = add i32 %mul.i171.1, %add105.1
+ %xor.i173.1 = xor i32 %add.i172.1, %mul.i171.1
+ %shr.i174.1 = lshr i32 %sub104.1, 15
+ %and.i175.1 = and i32 %shr.i174.1, 65537
+ %mul.i176.1 = mul nuw i32 %and.i175.1, 65535
+ %add.i177.1 = add i32 %mul.i176.1, %sub104.1
+ %xor.i178.1 = xor i32 %add.i177.1, %mul.i176.1
+ %shr.i179.1 = lshr i32 %sub106.1, 15
+ %and.i180.1 = and i32 %shr.i179.1, 65537
+ %mul.i181.1 = mul nuw i32 %and.i180.1, 65535
+ %add.i182.1 = add i32 %mul.i181.1, %sub106.1
+ %xor.i183.1 = xor i32 %add.i182.1, %mul.i181.1
+ %add108.1 = add i32 %xor.i173.1, %add113
+ %add110.1 = add i32 %add108.1, %xor.i.1
+ %add112.1 = add i32 %add110.1, %xor.i178.1
+ %add113.1 = add i32 %add112.1, %xor.i183.1
+ %add78.2 = add nsw i32 %sub51.1, %sub51
+ %sub86.2 = sub nsw i32 %sub51, %sub51.1
+ %add94.2 = add nsw i32 %sub51.3, %sub51.2
+ %sub102.2 = sub nsw i32 %sub51.2, %sub51.3
+ %add103.2 = add nsw i32 %add94.2, %add78.2
+ %sub104.2 = sub nsw i32 %add78.2, %add94.2
+ %add105.2 = add nsw i32 %sub102.2, %sub86.2
+ %sub106.2 = sub nsw i32 %sub86.2, %sub102.2
+ %shr.i.2 = lshr i32 %add103.2, 15
+ %and.i.2 = and i32 %shr.i.2, 65537
+ %mul.i.2 = mul nuw i32 %and.i.2, 65535
+ %add.i.2 = add i32 %mul.i.2, %add103.2
+ %xor.i.2 = xor i32 %add.i.2, %mul.i.2
+ %shr.i169.2 = lshr i32 %add105.2, 15
+ %and.i170.2 = and i32 %shr.i169.2, 65537
+ %mul.i171.2 = mul nuw i32 %and.i170.2, 65535
+ %add.i172.2 = add i32 %mul.i171.2, %add105.2
+ %xor.i173.2 = xor i32 %add.i172.2, %mul.i171.2
+ %shr.i174.2 = lshr i32 %sub104.2, 15
+ %and.i175.2 = and i32 %shr.i174.2, 65537
+ %mul.i176.2 = mul nuw i32 %and.i175.2, 65535
+ %add.i177.2 = add i32 %mul.i176.2, %sub104.2
+ %xor.i178.2 = xor i32 %add.i177.2, %mul.i176.2
+ %shr.i179.2 = lshr i32 %sub106.2, 15
+ %and.i180.2 = and i32 %shr.i179.2, 65537
+ %mul.i181.2 = mul nuw i32 %and.i180.2, 65535
+ %add.i182.2 = add i32 %mul.i181.2, %sub106.2
+ %xor.i183.2 = xor i32 %add.i182.2, %mul.i181.2
+ %add108.2 = add i32 %xor.i173.2, %add113.1
+ %add110.2 = add i32 %add108.2, %xor.i.2
+ %add112.2 = add i32 %add110.2, %xor.i178.2
+ %add113.2 = add i32 %add112.2, %xor.i183.2
+ %add78.3 = add nsw i32 %sub59.1, %sub59
+ %sub86.3 = sub nsw i32 %sub59, %sub59.1
+ %add94.3 = add nsw i32 %sub59.3, %sub59.2
+ %sub102.3 = sub nsw i32 %sub59.2, %sub59.3
+ %add103.3 = add nsw i32 %add94.3, %add78.3
+ %sub104.3 = sub nsw i32 %add78.3, %add94.3
+ %add105.3 = add nsw i32 %sub102.3, %sub86.3
+ %sub106.3 = sub nsw i32 %sub86.3, %sub102.3
+ %shr.i.3 = lshr i32 %add103.3, 15
+ %and.i.3 = and i32 %shr.i.3, 65537
+ %mul.i.3 = mul nuw i32 %and.i.3, 65535
+ %add.i.3 = add i32 %mul.i.3, %add103.3
+ %xor.i.3 = xor i32 %add.i.3, %mul.i.3
+ %shr.i169.3 = lshr i32 %add105.3, 15
+ %and.i170.3 = and i32 %shr.i169.3, 65537
+ %mul.i171.3 = mul nuw i32 %and.i170.3, 65535
+ %add.i172.3 = add i32 %mul.i171.3, %add105.3
+ %xor.i173.3 = xor i32 %add.i172.3, %mul.i171.3
+ %shr.i174.3 = lshr i32 %sub104.3, 15
+ %and.i175.3 = and i32 %shr.i174.3, 65537
+ %mul.i176.3 = mul nuw i32 %and.i175.3, 65535
+ %add.i177.3 = add i32 %mul.i176.3, %sub104.3
+ %xor.i178.3 = xor i32 %add.i177.3, %mul.i176.3
+ %shr.i179.3 = lshr i32 %sub106.3, 15
+ %and.i180.3 = and i32 %shr.i179.3, 65537
+ %mul.i181.3 = mul nuw i32 %and.i180.3, 65535
+ %add.i182.3 = add i32 %mul.i181.3, %sub106.3
+ %xor.i183.3 = xor i32 %add.i182.3, %mul.i181.3
+ %add108.3 = add i32 %xor.i173.3, %add113.2
+ %add110.3 = add i32 %add108.3, %xor.i.3
+ %add112.3 = add i32 %add110.3, %xor.i178.3
+ %add113.3 = add i32 %add112.3, %xor.i183.3
+ %conv118 = and i32 %add113.3, 65535
+ %shr = lshr i32 %add113.3, 16
+ %add119 = add nuw nsw i32 %conv118, %shr
+ %shr120 = lshr i32 %add119, 1
+ ret i32 %shr120
+}
>From 19326cd717713eb164fa5c8446066c2a4249ac74 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Tue, 12 Aug 2025 10:21:30 -0700
Subject: [PATCH 02/12] updated basic-strided-loads.ll
---
.../RISCV/basic-strided-loads.ll | 20 +++----------------
1 file changed, 3 insertions(+), 17 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
index 645dbc49269f0..0135d3c01d9f6 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
@@ -630,25 +630,11 @@ define void @rt_stride_widen_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
; CHECK-LABEL: define void @rt_stride_widen_no_reordering(
; CHECK-SAME: ptr [[PL:%.*]], i64 [[STRIDE:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[OFFSET0:%.*]] = mul nsw i64 [[STRIDE]], 0
-; CHECK-NEXT: [[OFFSET4:%.*]] = mul nsw i64 [[STRIDE]], 1
-; CHECK-NEXT: [[OFFSET8:%.*]] = mul nsw i64 [[STRIDE]], 2
-; CHECK-NEXT: [[OFFSET12:%.*]] = mul nsw i64 [[STRIDE]], 3
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET0]]
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET4]]
-; CHECK-NEXT: [[GEP_L8:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET8]]
-; CHECK-NEXT: [[GEP_L12:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET12]]
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 16
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 16
-; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L8]], align 16
-; CHECK-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[GEP_L12]], align 16
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 16 [[GEP_L0]], i64 [[TMP1]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
; CHECK-NEXT: store <16 x i8> [[TMP8]], ptr [[GEP_S0]], align 16
; CHECK-NEXT: ret void
;
>From a4bc06bc8c7a61fdc5641127abfd7e5544997802 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Tue, 12 Aug 2025 10:41:51 -0700
Subject: [PATCH 03/12] did a "todo".
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 207d9fc969f76..69f4c317aaeb4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13190,12 +13190,10 @@ void BoUpSLP::transformNodes() {
if (StridedCost < OriginalVecCost) {
// Strided load is more profitable than consecutive load + reverse -
// transform the node to strided load.
- // TODO: StrideTy =
- // DL->getIndexType(E.scalars.front()->getPointerOperand()->getType());
+ Type *StrideTy = DL->getIndexType(cast<LoadInst>(E.Scalars.front())
+ ->getPointerOperand()
+ ->getType());
StridedPtrInfo SPtrInfo;
- Type *StrideTy = Type::getIntNTy(
- SE->getContext(),
- DL->getTypeStoreSizeInBits(E.Scalars.front()->getType()));
SPtrInfo.StrideVal = ConstantInt::get(StrideTy, 1);
SPtrInfo.Ty = VecTy;
TreeEntryToStridedPtrInfoMap[&E] = SPtrInfo;
>From 56e18e893c3ee959a51d26058912b5475c0753c9 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Tue, 12 Aug 2025 12:22:28 -0700
Subject: [PATCH 04/12] addressed review comments.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 112 +++++++++---------
1 file changed, 53 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 69f4c317aaeb4..3e321625d45de 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1931,7 +1931,7 @@ class BoUpSLP {
// strided load.
FixedVectorType *Ty = nullptr;
};
- DenseMap<TreeEntry *, StridedPtrInfo> TreeEntryToStridedPtrInfoMap;
+ SmallDenseMap<TreeEntry *, StridedPtrInfo> TreeEntryToStridedPtrInfoMap;
public:
/// Tracks the state we can represent the loads in the given sequence.
@@ -2233,12 +2233,12 @@ class BoUpSLP {
bool analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps, Type *ElemTy,
Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
- StridedPtrInfo *SPtrInfo) const;
+ StridedPtrInfo &SPtrInfo) const;
bool analyzeConstantStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
- StridedPtrInfo *SPtrInfo, int64_t Diff,
+ StridedPtrInfo &SPtrInfo, int64_t Diff,
Value *Ptr0, Value *PtrN) const;
/// Checks if the given array of loads can be represented as a vectorized,
@@ -4498,7 +4498,7 @@ class BoUpSLP {
TreeEntry::EntryState getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
- SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo *SPtrInfo);
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo);
/// Maps a specific scalar to its tree entry(ies).
SmallDenseMap<Value *, SmallVector<TreeEntry *>> ScalarToTreeEntries;
@@ -6471,7 +6471,7 @@ static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
- StridedPtrInfo *SPtrInfo) const {
+ StridedPtrInfo &SPtrInfo) const {
// Group the pointers by constant offset.
DenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
OffsetToPointerOpIdxMap;
@@ -6481,15 +6481,15 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
return false;
const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(PtrSCEV);
+ if (!Add)
+ return false;
int64_t Offset = 0;
- if (Add) {
- for (int i = 0; i < (int)Add->getNumOperands(); ++i) {
- const SCEVConstant *SC = dyn_cast<SCEVConstant>(Add->getOperand(i));
- if (!SC)
- continue;
- Offset = (int64_t)(SC->getAPInt().getLimitedValue());
- break;
- }
+ for (int I : seq<int>(Add->getNumOperands())) {
+ auto *SC = dyn_cast<SCEVConstant>(Add->getOperand(I));
+ if (!SC)
+ continue;
+ Offset = SC->getAPInt().getSExtValue();
+ break;
}
OffsetToPointerOpIdxMap[Offset].first.push_back(Ptr);
OffsetToPointerOpIdxMap[Offset].second.push_back(Idx);
@@ -6512,17 +6512,17 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
!TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))
return false;
- SmallVector<int> SortedOffsetsV;
+ SmallVector<int64_t> SortedOffsetsV;
for (auto [K, V] : OffsetToPointerOpIdxMap) {
SortedOffsetsV.push_back(K);
}
- llvm::sort(SortedOffsetsV);
+ sort(SortedOffsetsV);
if (NumOffsets > 1) {
- int CommonDiff = SortedOffsetsV[1] - SortedOffsetsV[0];
- if ((CommonDiff) != 1)
+ int64_t CommonDiff = SortedOffsetsV[1] - SortedOffsetsV[0];
+ if (CommonDiff != 1)
return false;
- for (int i = 1; i < (int)SortedOffsetsV.size() - 1; ++i) {
- if (SortedOffsetsV[i + 1] - SortedOffsetsV[i] != CommonDiff)
+ for (int I : seq<int>(1, SortedOffsetsV.size() - 1)) {
+ if (SortedOffsetsV[I + 1] - SortedOffsetsV[I] != CommonDiff)
return false;
}
}
@@ -6542,31 +6542,29 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
unsigned NumCoeffs0 = Coeffs0.size();
if (NumCoeffs0 * NumOffsets != Sz)
return false;
- llvm::sort(Coeffs0);
+ sort(Coeffs0);
SmallVector<unsigned> SortedIndicesDraft;
SortedIndicesDraft.resize(Sz);
- auto updateSortedIndices =
+ auto UpdateSortedIndices =
[&](SmallVectorImpl<unsigned> &SortedIndicesForOffset,
SmallVectorImpl<unsigned> &IndicesInAllPointerOps,
int64_t OffsetNum) {
- unsigned Num = 0;
- for (unsigned Idx : SortedIndicesForOffset) {
+ for (const auto [Num, Idx] : enumerate(SortedIndicesForOffset)) {
SortedIndicesDraft[Num * NumOffsets + OffsetNum] =
IndicesInAllPointerOps[Idx];
- ++Num;
}
};
- updateSortedIndices(SortedIndicesForOffset0, IndicesInAllPointerOps0, 0);
+ UpdateSortedIndices(SortedIndicesForOffset0, IndicesInAllPointerOps0, 0);
SmallVector<int64_t> Coeffs;
SmallVector<unsigned> SortedIndicesForOffset;
- for (int i = 1; i < NumOffsets; ++i) {
+ for (int I : seq<int>(1, NumOffsets)) {
Coeffs.clear();
SortedIndicesForOffset.clear();
- int64_t Offset = SortedOffsetsV[i];
+ int64_t Offset = SortedOffsetsV[I];
SmallVector<Value *> &PointerOpsForOffset =
OffsetToPointerOpIdxMap[Offset].first;
SmallVector<unsigned> &IndicesInAllPointerOps =
@@ -6574,26 +6572,23 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
const SCEV *StrideWithinGroup = calculateRtStride(
PointerOpsForOffset, ElemTy, *DL, *SE, SortedIndicesForOffset, Coeffs);
- if ((!StrideWithinGroup) || StrideWithinGroup != Stride0) {
+ if ((!StrideWithinGroup) || StrideWithinGroup != Stride0)
return false;
- }
if (Coeffs.size() != NumCoeffs0)
return false;
- llvm::sort(Coeffs);
- for (unsigned i = 0; i < NumCoeffs0; ++i) {
- if (Coeffs[i] != Coeffs0[i])
+ sort(Coeffs);
+ for (int J : seq<int>(0, NumCoeffs0)) {
+ if (Coeffs[J] != Coeffs0[J])
return false;
}
- updateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, i);
+ UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, I);
}
SortedIndices.clear();
SortedIndices = SortedIndicesDraft;
- if (SPtrInfo) {
- SPtrInfo->StrideSCEV = Stride0;
- SPtrInfo->Ty = StridedLoadTy;
- }
+ SPtrInfo.StrideSCEV = Stride0;
+ SPtrInfo.Ty = StridedLoadTy;
return true;
}
@@ -6927,20 +6922,21 @@ isMaskedLoadCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
// Same as analyzeRtStrideCandidate, but for constant strides.
bool BoUpSLP::analyzeConstantStrideCandidate(
ArrayRef<Value *> PointerOps, Type *ElemTy, Align CommonAlignment,
- SmallVectorImpl<unsigned> &SortedIndices, StridedPtrInfo *SPtrInfo,
+ SmallVectorImpl<unsigned> &SortedIndices, StridedPtrInfo &SPtrInfo,
int64_t Diff, Value *Ptr0, Value *PtrN) const {
const unsigned Sz = PointerOps.size();
SmallVector<int64_t> SortedOffsetsFromBase;
SortedOffsetsFromBase.resize(Sz);
- for (unsigned i = 0; i < Sz; ++i) {
+ for (unsigned I : seq<int>(0, Sz)) {
Value *Ptr =
- SortedIndices.empty() ? PointerOps[i] : PointerOps[SortedIndices[i]];
- SortedOffsetsFromBase[i] =
+ SortedIndices.empty() ? PointerOps[I] : PointerOps[SortedIndices[I]];
+ SortedOffsetsFromBase[I] =
*getPointersDiff(ElemTy, Ptr0, ElemTy, Ptr, *DL, *SE);
}
// Find where the first group ends.
- assert(SortedOffsetsFromBase.size() > 1);
+ assert(SortedOffsetsFromBase.size() > 1 &&
+ "Trying to generate strided load for less than 2 loads");
int64_t StrideWithinGroup =
SortedOffsetsFromBase[1] - SortedOffsetsFromBase[0];
unsigned GroupSize = 1;
@@ -6987,7 +6983,7 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
if (CurrentGroupStartIdx != Sz)
return false;
- auto checkGroup = [&](unsigned StartIdx, unsigned GroupSize0,
+ auto CheckGroup = [&](unsigned StartIdx, unsigned GroupSize0,
int64_t StrideWithinGroup) -> bool {
unsigned GroupEndIdx = StartIdx + 1;
for (; GroupEndIdx != Sz; ++GroupEndIdx) {
@@ -6996,10 +6992,10 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
StrideWithinGroup)
break;
}
- return (GroupEndIdx - StartIdx == GroupSize0);
+ return GroupEndIdx - StartIdx == GroupSize0;
};
- for (unsigned i = 0; i < Sz; i += GroupSize) {
- if (!checkGroup(i, GroupSize, StrideWithinGroup))
+ for (unsigned I = 0; I < Sz; I += GroupSize) {
+ if (!CheckGroup(I, GroupSize, StrideWithinGroup))
return false;
}
}
@@ -7037,11 +7033,9 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
has_single_bit(AbsoluteDiff))) &&
AbsoluteDiff > VecSz) ||
Diff == -(static_cast<int>(VecSz) - 1)) {
- if (SPtrInfo) {
- Type *StrideTy = DL->getIndexType(Ptr0->getType());
- SPtrInfo->StrideVal = ConstantInt::get(StrideTy, StrideIntVal);
- SPtrInfo->Ty = StridedLoadTy;
- }
+ Type *StrideTy = DL->getIndexType(Ptr0->getType());
+ SPtrInfo.StrideVal = ConstantInt::get(StrideTy, StrideIntVal);
+ SPtrInfo.Ty = StridedLoadTy;
return true;
}
return false;
@@ -7089,7 +7083,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
if (!IsSorted) {
if (Sz > MinProfitableStridedLoads &&
analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order,
- SPtrInfo))
+ *SPtrInfo))
return LoadsState::StridedVectorize;
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
@@ -7124,7 +7118,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
return LoadsState::CompressVectorize;
// Simple check if not a strided access - clear order.
if (analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
- Order, SPtrInfo, *Diff, Ptr0, PtrN))
+ Order, *SPtrInfo, *Diff, Ptr0, PtrN))
return LoadsState::StridedVectorize;
}
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
@@ -9995,7 +9989,7 @@ getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
- SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo *SPtrInfo) {
+ SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo) {
assert(S.getMainOp() &&
"Expected instructions with same/alternate opcodes only.");
@@ -10097,7 +10091,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
});
});
};
- switch (canVectorizeLoads(VL, VL0, CurrentOrder, PointerOps, SPtrInfo)) {
+ switch (canVectorizeLoads(VL, VL0, CurrentOrder, PointerOps, &SPtrInfo)) {
case LoadsState::Vectorize:
return TreeEntry::Vectorize;
case LoadsState::CompressVectorize:
@@ -11531,7 +11525,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
TreeEntry::EntryState State = getScalarsVectorizationState(
- S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps, &SPtrInfo);
+ S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps, SPtrInfo);
if (State == TreeEntry::NeedToGather) {
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
@@ -19697,16 +19691,16 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Type *StrideTy = DL->getIndexType(PO->getType());
Value *StrideVal;
- StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap[E];
+ const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
StridedLoadTy = SPtrInfo.Ty;
- assert(StridedLoadTy);
+ assert(StridedLoadTy && "Missing StridedPoinerInfo for tree entry.");
unsigned StridedLoadEC =
StridedLoadTy->getElementCount().getKnownMinValue();
Value *Stride = SPtrInfo.StrideVal;
if (!Stride) {
const SCEV *StrideSCEV = SPtrInfo.StrideSCEV;
- assert(StrideSCEV);
+ assert(StrideSCEV && "Neither StrideVal nor StrideSCEV were set.");
SCEVExpander Expander(*SE, *DL, "strided-load-vec");
Stride = Expander.expandCodeFor(StrideSCEV, StrideSCEV->getType(),
&*Builder.GetInsertPoint());
>From 0728a05a8d8fb7ed6276ab801d3ddc62b157ec12 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 08:26:58 -0700
Subject: [PATCH 05/12] address review comments.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 34 ++++++++++++++-----
1 file changed, 25 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3e321625d45de..819d3a027d00c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2230,11 +2230,29 @@ class BoUpSLP {
/// may not be necessary.
bool isLoadCombineCandidate(ArrayRef<Value *> Stores) const;
+ // Suppose we are given pointers of the form: %b + x * %s + y * %c
+ // where %c is constant. Check if the pointers can be rearranged as follows:
+ // %b + 0 * %s + 0
+ // %b + 0 * %s + 1
+ // %b + 0 * %s + 2
+ // ...
+ // %b + 0 * %s + w
+ //
+ // %b + 1 * %s + 0
+ // %b + 1 * %s + 1
+ // %b + 1 * %s + 2
+ // ...
+ // %b + 1 * %s + w
+ // ...
+ //
+ // If the pointers can be rearanged in the above pattern, it means that the
+ // memory can be accessed with a strided loads of width `w` and stride `%s`.
bool analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps, Type *ElemTy,
Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
StridedPtrInfo &SPtrInfo) const;
+ // Same as analyzeRtStrideCandidate, but for constant strides.
bool analyzeConstantStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
@@ -6513,9 +6531,8 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
return false;
SmallVector<int64_t> SortedOffsetsV;
- for (auto [K, V] : OffsetToPointerOpIdxMap) {
+ for (auto [K, V] : OffsetToPointerOpIdxMap)
SortedOffsetsV.push_back(K);
- }
sort(SortedOffsetsV);
if (NumOffsets > 1) {
int64_t CommonDiff = SortedOffsetsV[1] - SortedOffsetsV[0];
@@ -6572,17 +6589,16 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
const SCEV *StrideWithinGroup = calculateRtStride(
PointerOpsForOffset, ElemTy, *DL, *SE, SortedIndicesForOffset, Coeffs);
- if ((!StrideWithinGroup) || StrideWithinGroup != Stride0)
+ if (!StrideWithinGroup || StrideWithinGroup != Stride0)
return false;
if (Coeffs.size() != NumCoeffs0)
return false;
sort(Coeffs);
- for (int J : seq<int>(0, NumCoeffs0)) {
- if (Coeffs[J] != Coeffs0[J])
- return false;
- }
+ if (Coeffs != Coeffs0)
+ return false
- UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, I);
+ UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps,
+ I);
}
SortedIndices.clear();
@@ -6927,7 +6943,7 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
const unsigned Sz = PointerOps.size();
SmallVector<int64_t> SortedOffsetsFromBase;
SortedOffsetsFromBase.resize(Sz);
- for (unsigned I : seq<int>(0, Sz)) {
+ for (unsigned I : seq<int>(Sz)) {
Value *Ptr =
SortedIndices.empty() ? PointerOps[I] : PointerOps[SortedIndices[I]];
SortedOffsetsFromBase[I] =
>From 33dee74ba137b765c67ce92bb6fdb7bc88cc78b7 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 08:56:01 -0700
Subject: [PATCH 06/12] added cost estimation
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 819d3a027d00c..d08c3b27466af 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6595,7 +6595,7 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
return false;
sort(Coeffs);
if (Coeffs != Coeffs0)
- return false
+ return false;
UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps,
I);
@@ -14992,11 +14992,19 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
break;
case TreeEntry::StridedVectorize: {
+ const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
+ FixedVectorType *StridedLoadTy = SPtrInfo.Ty;
+ assert(StridedLoadTy && "Missing StridedPoinerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
VecLdCost = TTI->getStridedMemoryOpCost(
- Instruction::Load, VecTy, LI0->getPointerOperand(),
+ Instruction::Load, StridedLoadTy, LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment, CostKind);
+ if (StridedLoadTy != VecTy)
+ VecLdCost +=
+ TTI->getCastInstrCost(Instruction::BitCast, StridedLoadTy, VecTy,
+ getCastContextHint(*E), CostKind);
+
break;
}
case TreeEntry::CompressVectorize: {
>From 0d4f1fd1a065a18332d154861fa07d3ca46de006 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 09:13:25 -0700
Subject: [PATCH 07/12] format
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d08c3b27466af..97b817d350061 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6597,8 +6597,7 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
if (Coeffs != Coeffs0)
return false;
- UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps,
- I);
+ UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, I);
}
SortedIndices.clear();
>From a7fcbfca178f727ecfe5490c77f80455ef5893f8 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 11:43:09 -0700
Subject: [PATCH 08/12] change references back to pointers because
canVectorizeLoads can be called with nullptr
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 24 +++++++++++--------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 97b817d350061..718073b01ef20 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2250,13 +2250,13 @@ class BoUpSLP {
bool analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps, Type *ElemTy,
Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
- StridedPtrInfo &SPtrInfo) const;
+ StridedPtrInfo *SPtrInfo) const;
// Same as analyzeRtStrideCandidate, but for constant strides.
bool analyzeConstantStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
- StridedPtrInfo &SPtrInfo, int64_t Diff,
+ StridedPtrInfo *SPtrInfo, int64_t Diff,
Value *Ptr0, Value *PtrN) const;
/// Checks if the given array of loads can be represented as a vectorized,
@@ -6489,7 +6489,7 @@ static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
- StridedPtrInfo &SPtrInfo) const {
+ StridedPtrInfo *SPtrInfo) const {
// Group the pointers by constant offset.
DenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
OffsetToPointerOpIdxMap;
@@ -6602,8 +6602,10 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
SortedIndices.clear();
SortedIndices = SortedIndicesDraft;
- SPtrInfo.StrideSCEV = Stride0;
- SPtrInfo.Ty = StridedLoadTy;
+ if (SPtrInfo) {
+ SPtrInfo->StrideSCEV = Stride0;
+ SPtrInfo->Ty = StridedLoadTy;
+ }
return true;
}
@@ -6937,7 +6939,7 @@ isMaskedLoadCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
// Same as analyzeRtStrideCandidate, but for constant strides.
bool BoUpSLP::analyzeConstantStrideCandidate(
ArrayRef<Value *> PointerOps, Type *ElemTy, Align CommonAlignment,
- SmallVectorImpl<unsigned> &SortedIndices, StridedPtrInfo &SPtrInfo,
+ SmallVectorImpl<unsigned> &SortedIndices, StridedPtrInfo *SPtrInfo,
int64_t Diff, Value *Ptr0, Value *PtrN) const {
const unsigned Sz = PointerOps.size();
SmallVector<int64_t> SortedOffsetsFromBase;
@@ -7049,8 +7051,10 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
AbsoluteDiff > VecSz) ||
Diff == -(static_cast<int>(VecSz) - 1)) {
Type *StrideTy = DL->getIndexType(Ptr0->getType());
- SPtrInfo.StrideVal = ConstantInt::get(StrideTy, StrideIntVal);
- SPtrInfo.Ty = StridedLoadTy;
+ if (SPtrInfo) {
+ SPtrInfo->StrideVal = ConstantInt::get(StrideTy, StrideIntVal);
+ SPtrInfo->Ty = StridedLoadTy;
+ }
return true;
}
return false;
@@ -7098,7 +7102,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
if (!IsSorted) {
if (Sz > MinProfitableStridedLoads &&
analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order,
- *SPtrInfo))
+ SPtrInfo))
return LoadsState::StridedVectorize;
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
@@ -7133,7 +7137,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
return LoadsState::CompressVectorize;
// Simple check if not a strided access - clear order.
if (analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
- Order, *SPtrInfo, *Diff, Ptr0, PtrN))
+ Order, SPtrInfo, *Diff, Ptr0, PtrN))
return LoadsState::StridedVectorize;
}
if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) ||
>From 13c99d2adc77901c89137d75ce380631e94d5430 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 11:52:28 -0700
Subject: [PATCH 09/12] three slashes for comments, deleted wrong comment.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 99 ++++++++++---------
1 file changed, 50 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 718073b01ef20..b0eafa21bab61 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1918,17 +1918,15 @@ class BoUpSLP {
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
- // If we decide to generate strided load / store, this struct contains all the
- // necessary info. It's fields are calculated by analyzeRtStrideCandidate and
- // analyzeConstantStrideCandidate. Note that Stride can be given either as a
- // SCEV or as a Value if it already exists.
- // To get the stride in bytes, StrideVal (or value obtained from StrideSCEV)
- // has to by multiplied by the size of element of FixedVectorType.
+ /// If we decide to generate strided load / store, this struct contains all
+ /// the necessary info. It's fields are calculated by analyzeRtStrideCandidate
+ /// and analyzeConstantStrideCandidate. Note that Stride can be given either
+ /// as a SCEV or as a Value if it already exists. To get the stride in bytes,
+ /// StrideVal (or value obtained from StrideSCEV) has to by multiplied by the
+ /// size of element of FixedVectorType.
struct StridedPtrInfo {
Value *StrideVal = nullptr;
const SCEV *StrideSCEV = nullptr;
- // Represents the ammount which needs to be added to the base pointer of
- // strided load.
FixedVectorType *Ty = nullptr;
};
SmallDenseMap<TreeEntry *, StridedPtrInfo> TreeEntryToStridedPtrInfoMap;
@@ -2230,29 +2228,29 @@ class BoUpSLP {
/// may not be necessary.
bool isLoadCombineCandidate(ArrayRef<Value *> Stores) const;
- // Suppose we are given pointers of the form: %b + x * %s + y * %c
- // where %c is constant. Check if the pointers can be rearranged as follows:
- // %b + 0 * %s + 0
- // %b + 0 * %s + 1
- // %b + 0 * %s + 2
- // ...
- // %b + 0 * %s + w
- //
- // %b + 1 * %s + 0
- // %b + 1 * %s + 1
- // %b + 1 * %s + 2
- // ...
- // %b + 1 * %s + w
- // ...
- //
- // If the pointers can be rearanged in the above pattern, it means that the
- // memory can be accessed with a strided loads of width `w` and stride `%s`.
+ /// Suppose we are given pointers of the form: %b + x * %s + y * %c
+ /// where %c is constant. Check if the pointers can be rearranged as follows:
+ /// %b + 0 * %s + 0
+ /// %b + 0 * %s + 1
+ /// %b + 0 * %s + 2
+ /// ...
+ /// %b + 0 * %s + w
+ ///
+ /// %b + 1 * %s + 0
+ /// %b + 1 * %s + 1
+ /// %b + 1 * %s + 2
+ /// ...
+ /// %b + 1 * %s + w
+ /// ...
+ ///
+ /// If the pointers can be rearanged in the above pattern, it means that the
+ /// memory can be accessed with a strided loads of width `w` and stride `%s`.
bool analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps, Type *ElemTy,
Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
StridedPtrInfo *SPtrInfo) const;
- // Same as analyzeRtStrideCandidate, but for constant strides.
+ /// Same as analyzeRtStrideCandidate, but for constant strides.
bool analyzeConstantStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
@@ -6469,23 +6467,23 @@ static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
return Stride;
}
-// Suppose we are given pointers of the form: %b + x * %s + y * %c
-// where %c is constant. Check if the pointers can be rearranged as follows:
-// %b + 0 * %s + 0
-// %b + 0 * %s + 1
-// %b + 0 * %s + 2
-// ...
-// %b + 0 * %s + w
-//
-// %b + 1 * %s + 0
-// %b + 1 * %s + 1
-// %b + 1 * %s + 2
-// ...
-// %b + 1 * %s + w
-// ...
-//
-// If the pointers can be rearanged in the above pattern, it means that the
-// memory can be accessed with a strided loads of width `w` and stride `%s`.
+/// Suppose we are given pointers of the form: %b + x * %s + y * %c
+/// where %c is constant. Check if the pointers can be rearranged as follows:
+/// %b + 0 * %s + 0
+/// %b + 0 * %s + 1
+/// %b + 0 * %s + 2
+/// ...
+/// %b + 0 * %s + w
+///
+/// %b + 1 * %s + 0
+/// %b + 1 * %s + 1
+/// %b + 1 * %s + 2
+/// ...
+/// %b + 1 * %s + w
+/// ...
+///
+/// If the pointers can be rearanged in the above pattern, it means that the
+/// memory can be accessed with a strided loads of width `w` and stride `%s`.
bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
@@ -6936,7 +6934,7 @@ isMaskedLoadCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
CompressMask, LoadVecTy);
}
-// Same as analyzeRtStrideCandidate, but for constant strides.
+/// Same as analyzeRtStrideCandidate, but for constant strides.
bool BoUpSLP::analyzeConstantStrideCandidate(
ArrayRef<Value *> PointerOps, Type *ElemTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices, StridedPtrInfo *SPtrInfo,
@@ -15001,12 +14999,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
VecLdCost = TTI->getStridedMemoryOpCost(
- Instruction::Load, StridedLoadTy, LI0->getPointerOperand(),
+ Instruction::Load, VecTy, LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment, CostKind);
- if (StridedLoadTy != VecTy)
- VecLdCost +=
- TTI->getCastInstrCost(Instruction::BitCast, StridedLoadTy, VecTy,
- getCastContextHint(*E), CostKind);
+ //VecLdCost = TTI->getStridedMemoryOpCost(
+ // Instruction::Load, StridedLoadTy, LI0->getPointerOperand(),
+ // /*VariableMask=*/false, CommonAlignment, CostKind);
+ //if (StridedLoadTy != VecTy)
+ // VecLdCost +=
+ // TTI->getCastInstrCost(Instruction::BitCast, StridedLoadTy, VecTy,
+ // getCastContextHint(*E), CostKind);
break;
}
>From 7f39c036c71f7a108120f0083ee7e4dcc97e335c Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 13:10:11 -0700
Subject: [PATCH 10/12] revert a change made by mistake.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b0eafa21bab61..55e108fc008b6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6497,15 +6497,15 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
return false;
const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(PtrSCEV);
- if (!Add)
- return false;
int64_t Offset = 0;
- for (int I : seq<int>(Add->getNumOperands())) {
- auto *SC = dyn_cast<SCEVConstant>(Add->getOperand(I));
- if (!SC)
- continue;
- Offset = SC->getAPInt().getSExtValue();
- break;
+ if (Add) {
+ for (int I : seq<int>(Add->getNumOperands())) {
+ const SCEVConstant *SC = dyn_cast<SCEVConstant>(Add->getOperand(I));
+ if (!SC)
+ continue;
+ Offset = SC->getAPInt().getSExtValue();
+ break;
+ }
}
OffsetToPointerOpIdxMap[Offset].first.push_back(Ptr);
OffsetToPointerOpIdxMap[Offset].second.push_back(Idx);
>From d02b0bc869c82181e74ae4bf616b85fe19a5d573 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 13:13:57 -0700
Subject: [PATCH 11/12] Another DenseMap -> SmallDenseMap
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 55e108fc008b6..e43e7785e55e3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6489,7 +6489,7 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
SmallVectorImpl<unsigned> &SortedIndices,
StridedPtrInfo *SPtrInfo) const {
// Group the pointers by constant offset.
- DenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
+ SmallDenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
OffsetToPointerOpIdxMap;
for (auto [Idx, Ptr] : enumerate(PointerOps)) {
const SCEV *PtrSCEV = SE->getSCEV(Ptr);
>From 1457e15f728c0c252141903f1cf4174517f79bfc Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 13 Aug 2025 13:21:53 -0700
Subject: [PATCH 12/12] put cost estimation back and make the order of type
operands correct.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e43e7785e55e3..a9aedee969cf9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14999,15 +14999,12 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
VecLdCost = TTI->getStridedMemoryOpCost(
- Instruction::Load, VecTy, LI0->getPointerOperand(),
+ Instruction::Load, StridedLoadTy, LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment, CostKind);
- //VecLdCost = TTI->getStridedMemoryOpCost(
- // Instruction::Load, StridedLoadTy, LI0->getPointerOperand(),
- // /*VariableMask=*/false, CommonAlignment, CostKind);
- //if (StridedLoadTy != VecTy)
- // VecLdCost +=
- // TTI->getCastInstrCost(Instruction::BitCast, StridedLoadTy, VecTy,
- // getCastContextHint(*E), CostKind);
+ if (StridedLoadTy != VecTy)
+ VecLdCost +=
+ TTI->getCastInstrCost(Instruction::BitCast, VecTy, StridedLoadTy,
+ getCastContextHint(*E), CostKind);
break;
}
More information about the llvm-commits
mailing list