[llvm] [SLP] Update analyzeRtStrideCandidate() to correctly handle revectorization (PR #191878)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 14:03:50 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/191878
>From 180ad879b30e3c6c9cd73ee0165948e5c1de764f Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Mon, 13 Apr 2026 11:57:26 -0700
Subject: [PATCH 1/2] [SLP][NFC] Add tests for runtime strided loads during
revectorization
---
.../SLPVectorizer/RISCV/revec-strided-load.ll | 101 ++++++++++++++++++
1 file changed, 101 insertions(+)
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
index 6d62659c2fef1..33c7858ee701d 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
@@ -20,6 +20,55 @@ entry:
ret void
}
+define void @widened_strided_load_runtime(ptr %in0, ptr %out0, i64 %stride) {
+; CHECK-LABEL: @widened_strided_load_runtime(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[IN1:%.*]] = getelementptr <8 x i8>, ptr [[IN0:%.*]], i64 [[STRIDE:%.*]]
+; CHECK-NEXT: [[L0:%.*]] = load <8 x i8>, ptr [[IN0]], align 2
+; CHECK-NEXT: [[L1:%.*]] = load <8 x i8>, ptr [[IN1]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <8 x i8> [[L0]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> [[L1]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: store <16 x i8> [[TMP2]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr <8 x i8>, ptr %in0, i64 %stride
+ %l0 = load <8 x i8>, ptr %in0, align 2
+ %l1 = load <8 x i8>, ptr %in1, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 8
+ store <8 x i8> %l0, ptr %out0, align 2
+ store <8 x i8> %l1, ptr %out1, align 2
+ ret void
+}
+
+; Base case of strided load, implicitly is widened
+define void @widened_strided_load_runtime_more_elements(ptr %in0, ptr %out0, i64 %stride) {
+; CHECK-LABEL: @widened_strided_load_runtime_more_elements(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE:%.*]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr align 2 [[IN0:%.*]], i64 [[TMP0]], <8 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr <2 x i8>, ptr %in0, i64 %stride
+ %in2 = getelementptr <2 x i8>, ptr %in1, i64 %stride
+ %in3 = getelementptr <2 x i8>, ptr %in2, i64 %stride
+ %l0 = load <2 x i8>, ptr %in0, align 2
+ %l1 = load <2 x i8>, ptr %in1, align 2
+ %l2 = load <2 x i8>, ptr %in2, align 2
+ %l3 = load <2 x i8>, ptr %in3, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 2
+ %out2 = getelementptr i8, ptr %out0, i64 4
+ %out3 = getelementptr i8, ptr %out0, i64 6
+ store <2 x i8> %l0, ptr %out0, align 2
+ store <2 x i8> %l1, ptr %out1, align 2
+ store <2 x i8> %l2, ptr %out2, align 2
+ store <2 x i8> %l3, ptr %out3, align 2
+ ret void
+}
+
; Widened strided load pattern but vectorized types
define void @doubly_widened_strided_load(ptr %in0, ptr %out0) {
; CHECK-LABEL: @doubly_widened_strided_load(
@@ -47,6 +96,36 @@ entry:
ret void
}
+define void @doubly_widened_strided_load_runtime(ptr %in0, ptr %out0, i64 %stride) {
+; CHECK-LABEL: @doubly_widened_strided_load_runtime(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[IN2:%.*]] = getelementptr <2 x i8>, ptr [[IN0:%.*]], i64 [[STRIDE:%.*]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[IN0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN2]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: store <8 x i8> [[TMP4]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr <2 x i8>, ptr %in0, i64 1
+ %in2 = getelementptr <2 x i8>, ptr %in0, i64 %stride
+ %in3 = getelementptr <2 x i8>, ptr %in2, i64 1
+ %l0 = load <2 x i8>, ptr %in0, align 2
+ %l1 = load <2 x i8>, ptr %in1, align 2
+ %l2 = load <2 x i8>, ptr %in2, align 2
+ %l3 = load <2 x i8>, ptr %in3, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 2
+ %out2 = getelementptr i8, ptr %out0, i64 4
+ %out3 = getelementptr i8, ptr %out0, i64 6
+ store <2 x i8> %l0, ptr %out0, align 2
+ store <2 x i8> %l1, ptr %out1, align 2
+ store <2 x i8> %l2, ptr %out2, align 2
+ store <2 x i8> %l3, ptr %out3, align 2
+ ret void
+}
+
; The resulting widened type isn't legal
define void @too_wide(ptr %in0, ptr %out0) {
; CHECK-LABEL: @too_wide(
@@ -70,6 +149,28 @@ entry:
ret void
}
+define void @too_wide_runtime(ptr %in0, ptr %out0, i64 %stride) {
+; CHECK-LABEL: @too_wide_runtime(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[IN1:%.*]] = getelementptr <8 x i16>, ptr [[IN0:%.*]], i64 [[STRIDE:%.*]]
+; CHECK-NEXT: [[L0:%.*]] = load <8 x i16>, ptr [[IN0]], align 2
+; CHECK-NEXT: [[L1:%.*]] = load <8 x i16>, ptr [[IN1]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <8 x i16> [[L0]], <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[L1]], <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP0]], <16 x i16> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: store <16 x i16> [[TMP2]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr <8 x i16>, ptr %in0, i64 %stride
+ %l0 = load <8 x i16>, ptr %in0, align 2
+ %l1 = load <8 x i16>, ptr %in1, align 2
+ %out1 = getelementptr i16, ptr %out0, i64 8
+ store <8 x i16> %l0, ptr %out0, align 2
+ store <8 x i16> %l1, ptr %out1, align 2
+ ret void
+}
+
; Stride size isn't aligned to the vector size
define void @non_aligned_stride(ptr %in0, ptr %out0) {
; CHECK-LABEL: @non_aligned_stride(
>From 634af4933182bfea8ed9fcaa0f52de540c18cf3f Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Mon, 13 Apr 2026 11:13:30 -0700
Subject: [PATCH 2/2] [SLP] Update analyzeRtStrideCandidate() to correctly
handle revectorization
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 11 +++++++---
.../SLPVectorizer/RISCV/revec-strided-load.ll | 21 +++++++------------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 87830329be1e2..ddeb7b428010a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7355,14 +7355,19 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
if (Sz % NumOffsets != 0)
return false;
VecSz = Sz / NumOffsets;
+ }
+ if (NumOffsets > 1 || ScalarTy->isVectorTy())
NewScalarTy = Type::getIntNTy(
SE->getContext(),
DL->getTypeSizeInBits(ScalarTy).getFixedValue() * NumOffsets);
- }
FixedVectorType *StridedLoadTy = getWidenedType(NewScalarTy, VecSz);
unsigned MinProfitableStridedOps =
IsLoad ? MinProfitableStridedLoads : MinProfitableStridedStores;
- if (Sz <= MinProfitableStridedOps || !TTI->isTypeLegal(StridedLoadTy) ||
+ unsigned InputTyNumElts = 1;
+ if (auto *FVT = dyn_cast<FixedVectorType>(ScalarTy))
+ InputTyNumElts = FVT->getNumElements();
+ if (Sz * InputTyNumElts <= MinProfitableStridedOps ||
+ !TTI->isTypeLegal(StridedLoadTy) ||
!TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))
return false;
@@ -7378,7 +7383,7 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
if (NumOffsets > 1) {
for (int I : seq<int>(1, SortedOffsetsV.size())) {
- if (SortedOffsetsV[I] - SortedOffsetsV[I - 1] != 1)
+ if (SortedOffsetsV[I] - SortedOffsetsV[I - 1] != InputTyNumElts)
return false;
}
}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
index 33c7858ee701d..3229a788e33d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
@@ -23,12 +23,9 @@ entry:
define void @widened_strided_load_runtime(ptr %in0, ptr %out0, i64 %stride) {
; CHECK-LABEL: @widened_strided_load_runtime(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IN1:%.*]] = getelementptr <8 x i8>, ptr [[IN0:%.*]], i64 [[STRIDE:%.*]]
-; CHECK-NEXT: [[L0:%.*]] = load <8 x i8>, ptr [[IN0]], align 2
-; CHECK-NEXT: [[L1:%.*]] = load <8 x i8>, ptr [[IN1]], align 2
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <8 x i8> [[L0]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> [[L1]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE:%.*]], 8
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 2 [[IN0:%.*]], i64 [[TMP0]], <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to <16 x i8>
; CHECK-NEXT: store <16 x i8> [[TMP2]], ptr [[OUT0:%.*]], align 2
; CHECK-NEXT: ret void
;
@@ -47,7 +44,8 @@ define void @widened_strided_load_runtime_more_elements(ptr %in0, ptr %out0, i64
; CHECK-LABEL: @widened_strided_load_runtime_more_elements(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE:%.*]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i64(ptr align 2 [[IN0:%.*]], i64 [[TMP0]], <8 x i1> splat (i1 true), i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i16> @llvm.experimental.vp.strided.load.v4i16.p0.i64(ptr align 2 [[IN0:%.*]], i64 [[TMP0]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[OUT0:%.*]], align 2
; CHECK-NEXT: ret void
;
@@ -99,12 +97,9 @@ entry:
define void @doubly_widened_strided_load_runtime(ptr %in0, ptr %out0, i64 %stride) {
; CHECK-LABEL: @doubly_widened_strided_load_runtime(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IN2:%.*]] = getelementptr <2 x i8>, ptr [[IN0:%.*]], i64 [[STRIDE:%.*]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[IN0]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN2]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE:%.*]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @llvm.experimental.vp.strided.load.v2i32.p0.i64(ptr align 2 [[IN0:%.*]], i64 [[TMP0]], <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP1]] to <8 x i8>
; CHECK-NEXT: store <8 x i8> [[TMP4]], ptr [[OUT0:%.*]], align 2
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list