[llvm] [SLP] Fix handling of strided loads during re-vectorization (PR #191294)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 9 15:12:49 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/191294
>From c036c3134b96efeb44ffe7ba33953eb3b4bb8e2d Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 9 Apr 2026 13:14:42 -0700
Subject: [PATCH 1/2] [SLP] Add tests for revectorization of strided loads
---
.../SLPVectorizer/RISCV/revec-strided-load.ll | 116 ++++++++++++++++++
1 file changed, 116 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
new file mode 100644
index 0000000000000..8c7ba2f980bf7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=riscv64 -mcpu=sifive-p870 -passes=slp-vectorizer -S -slp-revec -slp-threshold=-100 %s | FileCheck %s
+
+; Base case of strided load, implicitly is widened
+define void @widened_strided_load(ptr %in0, ptr %out0) {
+; CHECK-LABEL: @widened_strided_load(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr align 2 [[IN0:%.*]], i64 16, <16 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT: store <16 x i8> [[TMP0]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr i8, ptr %in0, i64 16
+ %l0 = load <8 x i8>, ptr %in0, align 2
+ %l1 = load <8 x i8>, ptr %in1, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 8
+ store <8 x i8> %l0, ptr %out0, align 2
+ store <8 x i8> %l1, ptr %out1, align 2
+ ret void
+}
+
+; Widened strided load pattern but vectorized types
+define void @doubly_widened_strided_load(ptr %in0, ptr %out0) {
+; CHECK-LABEL: @doubly_widened_strided_load(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @llvm.experimental.vp.strided.load.v2i32.p0.i64(ptr align 2 [[IN0:%.*]], i64 20, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
+; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr i8, ptr %in0, i64 2
+ %in2 = getelementptr i8, ptr %in0, i64 20
+ %in3 = getelementptr i8, ptr %in0, i64 22
+ %l0 = load <2 x i8>, ptr %in0, align 2
+ %l1 = load <2 x i8>, ptr %in1, align 2
+ %l2 = load <2 x i8>, ptr %in2, align 2
+ %l3 = load <2 x i8>, ptr %in3, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 2
+ %out2 = getelementptr i8, ptr %out0, i64 4
+ %out3 = getelementptr i8, ptr %out0, i64 6
+ store <2 x i8> %l0, ptr %out0, align 2
+ store <2 x i8> %l1, ptr %out1, align 2
+ store <2 x i8> %l2, ptr %out2, align 2
+ store <2 x i8> %l3, ptr %out3, align 2
+ ret void
+}
+
+; The resulting widened type isn't legal
+define void @too_wide(ptr %in0, ptr %out0) {
+; CHECK-LABEL: @too_wide(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i16> @llvm.experimental.vp.strided.load.v16i16.p0.i64(ptr align 2 [[IN0:%.*]], i64 32, <16 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT: store <16 x i16> [[TMP0]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr i16, ptr %in0, i64 16
+ %l0 = load <8 x i16>, ptr %in0, align 2
+ %l1 = load <8 x i16>, ptr %in1, align 2
+ %out1 = getelementptr i16, ptr %out0, i64 8
+ store <8 x i16> %l0, ptr %out0, align 2
+ store <8 x i16> %l1, ptr %out1, align 2
+ ret void
+}
+
+; Stride size isn't aligned to the vector size
+define void @non_aligned_stride(ptr %in0, ptr %out0) {
+; CHECK-LABEL: @non_aligned_stride(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[IN1:%.*]] = getelementptr i8, ptr [[IN0:%.*]], i64 3
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x ptr> poison, ptr [[IN0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x ptr> [[TMP0]], ptr [[IN1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, <4 x ptr> [[TMP2]], <4 x i64> <i64 0, i64 1, i64 0, i64 1>
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 2 [[TMP3]], <4 x i1> splat (i1 true), <4 x i8> poison)
+; CHECK-NEXT: store <4 x i8> [[TMP4]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr i8, ptr %in0, i64 3
+ %l0 = load <2 x i8>, ptr %in0, align 2
+ %l1 = load <2 x i8>, ptr %in1, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 2
+ store <2 x i8> %l0, ptr %out0, align 2
+ store <2 x i8> %l1, ptr %out1, align 2
+ ret void
+}
+
+; Stride size isn't aligned to the vector size
+; Scalar version of @non_aligned_stride
+define void @non_aligned_stride_scalar(ptr %in0, ptr %out0) {
+; CHECK-LABEL: @non_aligned_stride_scalar(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call <5 x i8> @llvm.masked.load.v5i8.p0(ptr align 2 [[IN0:%.*]], <5 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true>, <5 x i8> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <5 x i8> [[TMP0]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 3, i32 4>
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[OUT0:%.*]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %in1 = getelementptr i8, ptr %in0, i64 1
+ %in2 = getelementptr i8, ptr %in0, i64 3
+ %in3 = getelementptr i8, ptr %in0, i64 4
+ %l0 = load i8, ptr %in0, align 2
+ %l1 = load i8, ptr %in1, align 2
+ %l2 = load i8, ptr %in2, align 2
+ %l3 = load i8, ptr %in3, align 2
+ %out1 = getelementptr i8, ptr %out0, i64 1
+ %out2 = getelementptr i8, ptr %out0, i64 2
+ %out3 = getelementptr i8, ptr %out0, i64 3
+ store i8 %l0, ptr %out0, align 2
+ store i8 %l1, ptr %out1, align 2
+ store i8 %l2, ptr %out2, align 2
+ store i8 %l3, ptr %out3, align 2
+ ret void
+}
>From 5e84e83cddcdf461909f25e9a42b6d2a88a7a895 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Thu, 9 Apr 2026 12:52:55 -0700
Subject: [PATCH 2/2] [SLP] Fix handling of stride loads during
re-vectorization
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 7 ++++---
.../SLPVectorizer/RISCV/revec-strided-load.ll | 10 ++++++++--
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 16ef024561e26..7f649402b827d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7271,6 +7271,7 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
// Quick detour: at this point we can say what the type of strided load would
// be if all the checks pass. Check if this type is legal for the target.
bool NeedsWidening = Sz != GroupSize;
+ const uint64_t UnitBitWidth = DL->getTypeSizeInBits(ScalarTy).getFixedValue();
if (NeedsWidening) {
if (Sz % GroupSize != 0)
return false;
@@ -7278,9 +7279,9 @@ bool BoUpSLP::analyzeConstantStrideCandidate(
if (StrideWithinGroup != 1)
return false;
VecSz = Sz / GroupSize;
- NewScalarTy = Type::getIntNTy(
- SE->getContext(),
- DL->getTypeSizeInBits(ScalarTy).getFixedValue() * GroupSize);
+ NewScalarTy = Type::getIntNTy(SE->getContext(), UnitBitWidth * GroupSize);
+ } else if (ScalarTy->isVectorTy()) {
+ NewScalarTy = Type::getIntNTy(SE->getContext(), UnitBitWidth);
}
if (!isStridedLoad(PointerOps, NewScalarTy, Alignment, Diff, VecSz))
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
index 8c7ba2f980bf7..6d62659c2fef1 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-strided-load.ll
@@ -5,7 +5,8 @@
define void @widened_strided_load(ptr %in0, ptr %out0) {
; CHECK-LABEL: @widened_strided_load(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr align 2 [[IN0:%.*]], i64 16, <16 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i64(ptr align 2 [[IN0:%.*]], i64 16, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[TMP1]] to <16 x i8>
; CHECK-NEXT: store <16 x i8> [[TMP0]], ptr [[OUT0:%.*]], align 2
; CHECK-NEXT: ret void
;
@@ -50,7 +51,12 @@ entry:
define void @too_wide(ptr %in0, ptr %out0) {
; CHECK-LABEL: @too_wide(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i16> @llvm.experimental.vp.strided.load.v16i16.p0.i64(ptr align 2 [[IN0:%.*]], i64 32, <16 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT: [[IN1:%.*]] = getelementptr i16, ptr [[IN0:%.*]], i64 16
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x ptr> poison, ptr [[IN0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x ptr> [[TMP4]], ptr [[IN1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, <16 x ptr> [[TMP2]], <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr> align 2 [[TMP3]], <16 x i1> splat (i1 true), <16 x i16> poison)
; CHECK-NEXT: store <16 x i16> [[TMP0]], ptr [[OUT0:%.*]], align 2
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list