[llvm] 3572e62 - [SLPVectorizer] Widen rt stride loads (#162336)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 2 14:06:15 PST 2026
Author: Mikhail Gudim
Date: 2026-01-02T17:06:11-05:00
New Revision: 3572e62991f4936ab8be305480a992cde8117c47
URL: https://github.com/llvm/llvm-project/commit/3572e62991f4936ab8be305480a992cde8117c47
DIFF: https://github.com/llvm/llvm-project/commit/3572e62991f4936ab8be305480a992cde8117c47.diff
LOG: [SLPVectorizer] Widen rt stride loads (#162336)
Suppose we are given pointers of the form: `%b + x * %s + y * %c_i`
where `%c_i`s are constants and %s is a run-time fixed value.
If the pointers can be rearranged as follows:
```
%b + 0 * %s + 0
%b + 0 * %s + 1
%b + 0 * %s + 2
...
%b + 0 * %s + w
%b + 1 * %s + 0
%b + 1 * %s + 1
%b + 1 * %s + 2
...
%b + 1 * %s + w
...
```
It means that the memory can be accessed with a strided loads of width `w`
and stride `%s`.
This is motivated by x264 benchmark.
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5748d7ca752d2..e88fa2b8b6573 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6583,9 +6583,21 @@ static bool isReverseOrder(ArrayRef<unsigned> Order) {
/// Checks if the provided list of pointers \p Pointers represents the strided
/// pointers for type ElemTy. If they are not, nullptr is returned.
/// Otherwise, SCEV* of the stride value is returned.
+/// If `PointerOps` can be rearanged into the following sequence:
+/// ```
+/// %x + c_0 * stride,
+/// %x + c_1 * stride,
+/// %x + c_2 * stride
+/// ...
+/// ```
+/// where each `c_i` is constant. The `Coeffs` will contain `c_0, c_1, c_2, ..`
+/// and the SCEV of the `stride` will be returned.
static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
const DataLayout &DL, ScalarEvolution &SE,
- SmallVectorImpl<unsigned> &SortedIndices) {
+ SmallVectorImpl<unsigned> &SortedIndices,
+ SmallVectorImpl<int64_t> &Coeffs) {
+ assert(Coeffs.size() == PointerOps.size() &&
+ "Coeffs vector needs to be of correct size");
SmallVector<const SCEV *> SCEVs;
const SCEV *PtrSCEVLowest = nullptr;
const SCEV *PtrSCEVHighest = nullptr;
@@ -6650,7 +6662,7 @@ static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
std::set<DistOrdPair, decltype(Compare)> Offsets(Compare);
int Cnt = 0;
bool IsConsecutive = true;
- for (const SCEV *PtrSCEV : SCEVs) {
+ for (const auto [Idx, PtrSCEV] : enumerate(SCEVs)) {
unsigned Dist = 0;
if (PtrSCEV != PtrSCEVLowest) {
const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest);
@@ -6660,11 +6672,14 @@ static const SCEV *calculateRtStride(ArrayRef<Value *> PointerOps, Type *ElemTy,
const auto *SC = dyn_cast<SCEVConstant>(Coeff);
if (!SC || isa<SCEVCouldNotCompute>(SC))
return nullptr;
+ Coeffs[Idx] = (int64_t)SC->getAPInt().getLimitedValue();
if (!SE.getMinusSCEV(PtrSCEV, SE.getAddExpr(PtrSCEVLowest,
SE.getMulExpr(Stride, SC)))
->isZero())
return nullptr;
Dist = SC->getAPInt().getZExtValue();
+ } else {
+ Coeffs[Idx] = 0;
}
// If the strides are not the same or repeated, we can't vectorize.
if ((Dist / Size) * Size != Dist || (Dist / Size) >= SCEVs.size())
@@ -7162,18 +7177,203 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
Type *ScalarTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
StridedPtrInfo &SPtrInfo) const {
+ // If each value in `PointerOps` is of the form `%x + Offset` where `Offset`
+ // is constant, we partition `PointerOps` sequence into subsequences of
+ // pointers with the same offset. For each offset we record values from
+ // `PointerOps` and their indicies in `PointerOps`.
+ SmallDenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
+ OffsetToPointerOpIdxMap;
+ for (auto [Idx, Ptr] : enumerate(PointerOps)) {
+ const SCEV *PtrSCEV = SE->getSCEV(Ptr);
+ if (!PtrSCEV)
+ return false;
+
+ const auto *Add = dyn_cast<SCEVAddExpr>(PtrSCEV);
+ int64_t Offset = 0;
+ if (Add) {
+ // `Offset` is non-zero.
+ for (int I : seq<int>(Add->getNumOperands())) {
+ const auto *SC = dyn_cast<SCEVConstant>(Add->getOperand(I));
+ if (!SC)
+ continue;
+ Offset = SC->getAPInt().getSExtValue();
+ break;
+ }
+ }
+ OffsetToPointerOpIdxMap[Offset].first.push_back(Ptr);
+ OffsetToPointerOpIdxMap[Offset].second.push_back(Idx);
+ }
+ unsigned NumOffsets = OffsetToPointerOpIdxMap.size();
+
+ // Quick detour: at this point we can say what the type of strided load would
+ // be if all the checks pass. Check if this type is legal for the target.
const unsigned Sz = PointerOps.size();
- FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, Sz);
+ unsigned VecSz = Sz;
+ Type *NewScalarTy = ScalarTy;
+ if (NumOffsets > 1) {
+ if (Sz % NumOffsets != 0)
+ return false;
+ VecSz = Sz / NumOffsets;
+ NewScalarTy = Type::getIntNTy(
+ SE->getContext(),
+ DL->getTypeSizeInBits(ScalarTy).getFixedValue() * NumOffsets);
+ }
+ FixedVectorType *StridedLoadTy = getWidenedType(NewScalarTy, VecSz);
if (Sz <= MinProfitableStridedLoads || !TTI->isTypeLegal(StridedLoadTy) ||
!TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))
return false;
- if (const SCEV *Stride =
- calculateRtStride(PointerOps, ScalarTy, *DL, *SE, SortedIndices)) {
- SPtrInfo.Ty = getWidenedType(ScalarTy, PointerOps.size());
- SPtrInfo.StrideSCEV = Stride;
- return true;
+
+ // Check if the offsets are contiguous and that each group has the required
+ // size.
+ SmallVector<int64_t> SortedOffsetsV(NumOffsets);
+ for (auto [Idx, MapPair] : enumerate(OffsetToPointerOpIdxMap)) {
+ if (MapPair.second.first.size() != VecSz)
+ return false;
+ SortedOffsetsV[Idx] = MapPair.first;
}
- return false;
+ sort(SortedOffsetsV);
+
+ if (NumOffsets > 1) {
+ for (int I : seq<int>(1, SortedOffsetsV.size())) {
+ if (SortedOffsetsV[I] - SortedOffsetsV[I - 1] != 1)
+ return false;
+ }
+ }
+
+ // Introduce some notation for the explanations below. Let `PointerOps_j`
+ // denote the subsequence of `PointerOps` with offsets equal to
+ // `SortedOffsetsV[j]`. Let `SortedIndices_j` be a such that the sequence
+ // ```
+ // PointerOps_j[SortedIndices_j[0]],
+ // PointerOps_j[SortedIndices_j[1]],
+ // PointerOps_j[SortedIndices_j[2]],
+ // ...
+ // ```
+ // is sorted. Also, let `IndicesInAllPointerOps_j` be the vector
+ // of indices of the subsequence `PointerOps_j` in all of `PointerOps`,
+ // i.e `PointerOps_j[i] = PointerOps[IndicesInAllPointerOps_j[i]]`.
+ // The entire sorted `PointerOps` looks like this:
+ // ```
+ // PointerOps_0[SortedIndices_0[0]] = PointerOps[IndicesInAllPointerOps_0[0]],
+ // PointerOps_1[SortedIndices_1[0]] = PointerOps[IndicesInAllPointerOps_1[0]],
+ // PointerOps_2[SortedIndices_2[0]] = PointerOps[IndicesInAllPointerOps_2[0]],
+ // ...
+ // PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[0]] =
+ // PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[0]],
+ //
+ // PointerOps_0[SortedIndices_0[1]] = PointerOps[IndicesInAllPointerOps_0[1]],
+ // PointerOps_1[SortedIndices_1[1]] = PointerOps[IndicesInAllPointerOps_1[1]],
+ // PointerOps_2[SortedIndices_2[1]] = PointerOps[IndicesInAllPointerOps_2[1]],
+ // ...
+ // PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[1]] =
+ // PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[1]],
+ //
+ // PointerOps_0[SortedIndices_0[2]] = PointerOps[IndicesInAllPointerOps_0[2]],
+ // PointerOps_1[SortedIndices_1[2]] = PointerOps[IndicesInAllPointerOps_1[2]],
+ // PointerOps_2[SortedIndices_2[2]] = PointerOps[IndicesInAllPointerOps_2[2]],
+ // ...
+ // PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[2]] =
+ // PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[2]],
+ // ...
+ // ...
+ // ...
+ // PointerOps_0[SortedIndices_0[VecSz - 1]] =
+ // PointerOps[IndicesInAllPointerOps_0[VecSz - 1]],
+ // PointerOps_1[SortedIndices_1[VecSz - 1]] =
+ // PointerOps[IndicesInAllPointerOps_1[VecSz - 1]],
+ // PointerOps_2[SortedIndices_2[VecSz - 1]] =
+ // PointerOps[IndicesInAllPointerOps_2[VecSz - 1]],
+ // ...
+ // PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[VecSz - 1]] =
+ // PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[VecSz - 1]],
+ // ```
+ // In order to be able to generate a strided load, we need the following
+ // checks to pass:
+ //
+ // (1) for each `PointerOps_j` check that the distance
+ // between adjacent pointers are all equal to the same value (stride).
+ // (2) for each `PointerOps_j` check that coefficients calculated by
+ // `calculateRtStride` are all the same.
+ //
+ // As we do that, also calculate SortedIndices. Since we should not modify
+ // `SortedIndices` unless we know that all the checks succeed, record the
+ // indicies into `SortedIndicesDraft`.
+ SmallVector<unsigned> SortedIndicesDraft(Sz);
+
+ // Given sorted indices for a particular offset (as calculated by
+ // calculateRtStride), update the `SortedIndicesDraft` for all of PointerOps.
+ // Let `Offset` be `SortedOffsetsV[OffsetNum]`.
+ // \param `OffsetNum` the index of `Offset` in `SortedOffsetsV`.
+ // \param `IndicesInAllPointerOps` vector of indices of the
+ // subsequence `PointerOps_OffsetNum` in `PointerOps`, i.e. using the above
+ // notation `IndicesInAllPointerOps = IndicesInAllPointerOps_OffsetNum`.
+ // \param `SortedIndicesForOffset = SortedIndices_OffsetNum`
+ auto UpdateSortedIndices =
+ [&](SmallVectorImpl<unsigned> &SortedIndicesForOffset,
+ ArrayRef<unsigned> IndicesInAllPointerOps, const int64_t OffsetNum) {
+ if (SortedIndicesForOffset.empty()) {
+ SortedIndicesForOffset.resize(IndicesInAllPointerOps.size());
+ std::iota(SortedIndicesForOffset.begin(),
+ SortedIndicesForOffset.end(), 0);
+ }
+ for (const auto [Num, Idx] : enumerate(SortedIndicesForOffset)) {
+ SortedIndicesDraft[Num * NumOffsets + OffsetNum] =
+ IndicesInAllPointerOps[Idx];
+ }
+ };
+
+ int64_t LowestOffset = SortedOffsetsV[0];
+ ArrayRef<Value *> PointerOps0 = OffsetToPointerOpIdxMap[LowestOffset].first;
+
+ SmallVector<int64_t> Coeffs0(VecSz);
+ SmallVector<unsigned> SortedIndicesForOffset0;
+ const SCEV *Stride0 = calculateRtStride(PointerOps0, ScalarTy, *DL, *SE,
+ SortedIndicesForOffset0, Coeffs0);
+ if (!Stride0)
+ return false;
+ unsigned NumCoeffs0 = Coeffs0.size();
+ if (NumCoeffs0 * NumOffsets != Sz)
+ return false;
+ sort(Coeffs0);
+
+ ArrayRef<unsigned> IndicesInAllPointerOps0 =
+ OffsetToPointerOpIdxMap[LowestOffset].second;
+ UpdateSortedIndices(SortedIndicesForOffset0, IndicesInAllPointerOps0, 0);
+
+ // Now that we know what the common stride and coefficients has to be check
+ // the remaining `PointerOps_j`.
+ SmallVector<int64_t> Coeffs;
+ SmallVector<unsigned> SortedIndicesForOffset;
+ for (int J : seq<int>(1, NumOffsets)) {
+ Coeffs.clear();
+ Coeffs.resize(VecSz);
+ SortedIndicesForOffset.clear();
+
+ int64_t Offset = SortedOffsetsV[J];
+ ArrayRef<Value *> PointerOpsForOffset =
+ OffsetToPointerOpIdxMap[Offset].first;
+ ArrayRef<unsigned> IndicesInAllPointerOps =
+ OffsetToPointerOpIdxMap[Offset].second;
+ const SCEV *StrideWithinGroup =
+ calculateRtStride(PointerOpsForOffset, ScalarTy, *DL, *SE,
+ SortedIndicesForOffset, Coeffs);
+
+ if (!StrideWithinGroup || StrideWithinGroup != Stride0)
+ return false;
+ if (Coeffs.size() != NumCoeffs0)
+ return false;
+ sort(Coeffs);
+ if (Coeffs != Coeffs0)
+ return false;
+
+ UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, J);
+ }
+
+ SortedIndices.clear();
+ SortedIndices = SortedIndicesDraft;
+ SPtrInfo.StrideSCEV = Stride0;
+ SPtrInfo.Ty = StridedLoadTy;
+ return true;
}
BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
index f3f9191a6fdc7..89ecaf9bb08f1 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
@@ -710,25 +710,11 @@ define void @rt_stride_widen_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
; CHECK-LABEL: define void @rt_stride_widen_no_reordering(
; CHECK-SAME: ptr [[PL:%.*]], i64 [[STRIDE:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[OFFSET0:%.*]] = mul nsw i64 [[STRIDE]], 0
-; CHECK-NEXT: [[OFFSET4:%.*]] = mul nsw i64 [[STRIDE]], 1
-; CHECK-NEXT: [[OFFSET8:%.*]] = mul nsw i64 [[STRIDE]], 2
-; CHECK-NEXT: [[OFFSET12:%.*]] = mul nsw i64 [[STRIDE]], 3
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET0]]
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET4]]
-; CHECK-NEXT: [[GEP_L8:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET8]]
-; CHECK-NEXT: [[GEP_L12:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET12]]
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L8]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[GEP_L12]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[GEP_L0]], i64 [[TMP1]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
; CHECK-NEXT: store <16 x i8> [[TMP8]], ptr [[GEP_S0]], align 1
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
index c1042f1842107..33249a8e66657 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/x264-satd-8x4.ll
@@ -10,66 +10,22 @@ define i32 @x264_pixel_satd_8x4(ptr %pix1, i32 %i_pix1, ptr %pix2, i32 %i_pix
; CHECK-NEXT: [[IDX_EXT63:%.*]] = sext i32 [[I_PIX2]] to i64
; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[PIX1]], i64 4
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[PIX2]], i64 4
-; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[PIX1]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[PIX2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
-; CHECK-NEXT: [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
-; CHECK-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_1]], i64 4
-; CHECK-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_1]], i64 4
-; CHECK-NEXT: [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
-; CHECK-NEXT: [[ARRAYIDX3_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_2]], i64 4
-; CHECK-NEXT: [[ARRAYIDX5_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_2]], i64 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PIX1]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[PIX2]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ADD_PTR]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i8>, ptr [[ADD_PTR64]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load <4 x i8>, ptr [[ADD_PTR_1]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_1]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
-; CHECK-NEXT: [[TMP12:%.*]] = load <4 x i8>, ptr [[ADD_PTR_2]], align 1
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP15]], <16 x i8> [[TMP16]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <16 x i8> [[TMP17]], <16 x i8> [[TMP18]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[IDX_EXT]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[PIX1]], i64 [[TMP0]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast <4 x i32> [[TMP1]] to <16 x i8>
; CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[TMP19]] to <16 x i32>
-; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_2]], align 1
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <16 x i8> [[TMP24]], <16 x i8> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP21]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP26]], <16 x i8> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[IDX_EXT63]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[PIX2]], i64 [[TMP4]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP28:%.*]] = bitcast <4 x i32> [[TMP5]] to <16 x i8>
; CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[TMP28]] to <16 x i32>
; CHECK-NEXT: [[TMP30:%.*]] = sub nsw <16 x i32> [[TMP20]], [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_3]], align 1
-; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x i8> [[TMP6]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP10]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <16 x i8> [[TMP34]], <16 x i8> [[TMP35]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP31]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <16 x i8> [[TMP36]], <16 x i8> [[TMP37]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[IDX_EXT]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[ARRAYIDX3]], i64 [[TMP9]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast <4 x i32> [[TMP10]] to <16 x i8>
; CHECK-NEXT: [[TMP39:%.*]] = zext <16 x i8> [[TMP38]] to <16 x i32>
-; CHECK-NEXT: [[TMP40:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
-; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP45:%.*]] = shufflevector <16 x i8> [[TMP43]], <16 x i8> [[TMP44]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP40]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <16 x i8> [[TMP45]], <16 x i8> [[TMP46]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[IDX_EXT63]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 1 [[ARRAYIDX5]], i64 [[TMP13]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP47:%.*]] = bitcast <4 x i32> [[TMP14]] to <16 x i8>
; CHECK-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[TMP47]] to <16 x i32>
; CHECK-NEXT: [[TMP49:%.*]] = sub nsw <16 x i32> [[TMP39]], [[TMP48]]
; CHECK-NEXT: [[TMP50:%.*]] = shl nsw <16 x i32> [[TMP49]], splat (i32 16)
More information about the llvm-commits
mailing list