[llvm] [SLP] Enable runtime stride stores (PR #200018)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 09:58:42 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/200018
>From b7a11c879e907a8b4919dd501e14fc4feb6965cf Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 22 Apr 2026 12:54:25 -0700
Subject: [PATCH 01/14] [SLP] Precommit tests for runtime strided stores
---
.../RISCV/runtime-strided-stores.ll | 995 ++++++++++++++++++
1 file changed, 995 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
new file mode 100644
index 0000000000000..0d5b1271bc86e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
@@ -0,0 +1,995 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+
+; RUN: opt -mtriple=riscv64 -mattr=+m,+v -passes=slp-vectorizer -S -slp-enable-strided-stores -slp-threshold=-100 -slp-revec < %s | FileCheck %s
+
+define void @runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 %stride
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @runtime_stride_complex_scev(ptr %pl, ptr %ps, i64 %stride, i64 %stride1) {
+; CHECK-LABEL: define void @runtime_stride_complex_scev(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]], i64 [[STRIDE1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_SS0:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS1:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS2:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS3:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS4:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS5:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS6:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_SS7:%.*]] = getelementptr i8, ptr [[GEP_S7]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_SS0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_SS1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_SS2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_SS3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_SS4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_SS5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_SS6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_SS7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 %stride
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ %gep_ss0 = getelementptr i8, ptr %gep_s0, i64 %stride1
+ %gep_ss1 = getelementptr i8, ptr %gep_s1, i64 %stride1
+ %gep_ss2 = getelementptr i8, ptr %gep_s2, i64 %stride1
+ %gep_ss3 = getelementptr i8, ptr %gep_s3, i64 %stride1
+ %gep_ss4 = getelementptr i8, ptr %gep_s4, i64 %stride1
+ %gep_ss5 = getelementptr i8, ptr %gep_s5, i64 %stride1
+ %gep_ss6 = getelementptr i8, ptr %gep_s6, i64 %stride1
+ %gep_ss7 = getelementptr i8, ptr %gep_s7, i64 %stride1
+
+ store i8 %load0, ptr %gep_ss0
+ store i8 %load1, ptr %gep_ss1
+ store i8 %load2, ptr %gep_ss2
+ store i8 %load3, ptr %gep_ss3
+ store i8 %load4, ptr %gep_ss4
+ store i8 %load5, ptr %gep_ss5
+ store i8 %load6, ptr %gep_ss6
+ store i8 %load7, ptr %gep_ss7
+
+ ret void
+}
+
+define void @two_runtime_strides(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @two_runtime_strides(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[PS1:%.*]] = mul i64 [[STRIDE]], 16
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 [[PS1]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %ps1 = mul i64 %stride, 16
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %ps, i64 %ps1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @runtime_strides_constant_gap(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_strides_constant_gap(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 100
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %ps, i64 100
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @overlapping_strides(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @overlapping_strides(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[STRIDE2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[STRIDE1:%.*]] = mul i64 [[STRIDE]], 2
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %stride1 = mul i64 %stride, 1
+ %ps1 = mul i64 %stride, 2
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride1
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride1
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride1
+ %gep_s4 = getelementptr i8, ptr %ps, i64 %ps1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride1
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride1
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride1
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride_unit_stride(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[GEP_L0]], <8 x i1> <i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 1
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 1
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[LOAD2:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
+; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[LOAD7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
+; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 1
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 1
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 1
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @unit_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @unit_stride_runtime_stride(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 1
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 1
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 1
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 1
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 1
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 1
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @runtime_stride_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride_constant_stride(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 2, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 2
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 2
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 2
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @constant_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @constant_stride_runtime_stride(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 2
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 2
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 2
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 2, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 2
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 2
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 2
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @overlap(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @overlap(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 1
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 1
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 1
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+
+ ret void
+}
+
+define void @overlap_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @overlap_constant_stride(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S3]], i64 2, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 2
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 2
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 2
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+
+ ret void
+}
+
+define void @runtime_stride_reorder(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride_reorder(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 2
+ %gep_l2 = getelementptr i8, ptr %pl, i64 1
+ %gep_l3 = getelementptr i8, ptr %pl, i64 4
+ %gep_l4 = getelementptr i8, ptr %pl, i64 7
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 3
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 %stride
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride_unschedulable(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: store i8 [[TMP10]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
+; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
+; CHECK-NEXT: store i8 [[TMP11]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 %stride
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i8 %load1, ptr %gep_s6
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i8 %load5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s1
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
+
+define void @runtime_stride_revec(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride_revec(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 4
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: store <4 x i8> [[TMP2]], ptr [[GEP_S0]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[GEP_S4]], align 4
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+
+ %load0 = load <4 x i8>, ptr %gep_l0
+ %load4 = load <4 x i8>, ptr %gep_l4
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s4 = getelementptr i8, ptr %ps, i64 %stride
+
+ store <4 x i8> %load0, ptr %gep_s0
+ store <4 x i8> %load4, ptr %gep_s4
+
+ ret void
+}
+
+define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
+; CHECK-LABEL: define void @runtime_stride_diff_types(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP1:%.*]] = call <5 x i8> @llvm.masked.load.v5i8.p0(ptr align 1 [[GEP_L0]], <5 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true>, <5 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <5 x i8> [[TMP1]], <5 x i8> poison, <4 x i32> <i32 0, i32 2, i32 3, i32 4>
+; CHECK-NEXT: [[LOAD6:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_L1]], i64 4, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
+; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i16
+; CHECK-NEXT: store i16 [[TMP6]], ptr [[GEP_S1]], align 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[TMP10]] to i16
+; CHECK-NEXT: store i16 [[TMP11]], ptr [[GEP_S5]], align 2
+; CHECK-NEXT: store i8 [[LOAD6]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: ret void
+;
+ %gep_l0 = getelementptr i8, ptr %pl, i64 0
+ %gep_l1 = getelementptr i8, ptr %pl, i64 1
+ %gep_l2 = getelementptr i8, ptr %pl, i64 2
+ %gep_l3 = getelementptr i8, ptr %pl, i64 3
+ %gep_l4 = getelementptr i8, ptr %pl, i64 4
+ %gep_l5 = getelementptr i8, ptr %pl, i64 5
+ %gep_l6 = getelementptr i8, ptr %pl, i64 6
+ %gep_l7 = getelementptr i8, ptr %pl, i64 7
+
+ %load0 = load i8, ptr %gep_l0
+ %load1 = load i8, ptr %gep_l1
+ %load2 = load i8, ptr %gep_l2
+ %load3 = load i8, ptr %gep_l3
+ %load4 = load i8, ptr %gep_l4
+ %load5 = load i8, ptr %gep_l5
+ %load6 = load i8, ptr %gep_l6
+ %load7 = load i8, ptr %gep_l7
+
+ %zext1 = zext i8 %load1 to i16
+ %zext5 = zext i8 %load5 to i16
+
+ %gep_s0 = getelementptr i8, ptr %ps, i64 0
+ %gep_s1 = getelementptr i8, ptr %gep_s0, i64 %stride
+ %gep_s2 = getelementptr i8, ptr %gep_s1, i64 %stride
+ %gep_s3 = getelementptr i8, ptr %gep_s2, i64 %stride
+ %gep_s4 = getelementptr i8, ptr %gep_s3, i64 %stride
+ %gep_s5 = getelementptr i8, ptr %gep_s4, i64 %stride
+ %gep_s6 = getelementptr i8, ptr %gep_s5, i64 %stride
+ %gep_s7 = getelementptr i8, ptr %gep_s6, i64 %stride
+
+ store i8 %load0, ptr %gep_s0
+ store i16 %zext1, ptr %gep_s1
+ store i8 %load2, ptr %gep_s2
+ store i8 %load3, ptr %gep_s3
+ store i8 %load4, ptr %gep_s4
+ store i16 %zext5, ptr %gep_s5
+ store i8 %load6, ptr %gep_s6
+ store i8 %load7, ptr %gep_s7
+
+ ret void
+}
>From c7ddf6b6b826b12a54cf49148fbe0ff2913f6152 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 27 May 2026 09:56:50 -0700
Subject: [PATCH 02/14] [SLP] Enable runtime stride stores
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 145 ++++++--
.../RISCV/runtime-strided-stores.ll | 310 +++++-------------
2 files changed, 198 insertions(+), 257 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b5ccccb2d0208..600bf005782c9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5347,10 +5347,15 @@ class slpvectorizer::BoUpSLP {
public:
ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
+ ScheduleBundle(bool IsValid, bool NoInternalAlias)
+ : ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid),
+ NoInternalAlias(NoInternalAlias) {}
static bool classof(const ScheduleEntity *Entity) {
return Entity->getKind() == Kind::ScheduleBundle;
}
+ bool NoInternalAlias = false;
+
/// Verify basic self consistency properties
void verify() const {
for (const ScheduleEntity *SD : Bundle) {
@@ -5369,6 +5374,14 @@ class slpvectorizer::BoUpSLP {
}
}
+ /// Check to see if an instruction is present in this bundle
+ bool findInst(Instruction *I) {
+ return std::find_if(Bundle.begin(), Bundle.end(),
+ [&](ScheduleEntity *SE) {
+ return SE->getInst() == I;
+ }) != Bundle.end();
+ }
+
/// Returns the number of unscheduled dependencies in the bundle.
int unscheduledDepsInBundle() const {
assert(*this && "bundle must not be empty");
@@ -6259,8 +6272,11 @@ class slpvectorizer::BoUpSLP {
/// \param VL The list of scalar instructions.
/// \param S The state of the instructions.
/// \param EI The edge in the SLP graph or the user node/operand number.
+ /// \param NoInternalAlias Ignore any schedule issues due to aliasing within
+ /// the bundle.
ScheduleBundle &buildBundle(ArrayRef<Value *> VL,
- const InstructionsState &S, const EdgeInfo &EI);
+ const InstructionsState &S, const EdgeInfo &EI,
+ bool NoInternalAlias);
/// Checks if a bundle of instructions can be scheduled, i.e. has no
/// cyclic dependencies. This is only a dry-run, no instructions are
@@ -6269,7 +6285,8 @@ class slpvectorizer::BoUpSLP {
/// std::nullopt if \p VL is allowed to be scheduled.
std::optional<ScheduleBundle *>
tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
- const InstructionsState &S, const EdgeInfo &EI);
+ const InstructionsState &S, const EdgeInfo &EI,
+ bool NoInternalAlias);
/// Allocates schedule data chunk.
ScheduleData *allocateScheduleDataChunks();
@@ -11035,6 +11052,8 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
PointerOps.push_back(SI->getPointerOperand());
}
+ Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
+
// Check the order of pointer operands.
if (llvm::sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, CurrentOrder)) {
Value *Ptr0;
@@ -11046,7 +11065,6 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
Ptr0 = PointerOps[CurrentOrder.front()];
PtrN = PointerOps[CurrentOrder.back()];
}
- Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
std::optional<int64_t> Dist =
getPointersDiff(ScalarTy, Ptr0, ScalarTy, PtrN, *DL, *SE);
// Check that the sorted pointer operands are consecutive.
@@ -11057,6 +11075,17 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
CurrentOrder, *Dist, Ptr0, SPtrInfo))
return TreeEntry::StridedVectorize;
}
+ if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
+ CurrentOrder, SPtrInfo, /*isLoad=*/false)) {
+ if (isIdentityOrder(CurrentOrder)) {
+ CurrentOrder.clear();
+ } else if (isReverseOrder(CurrentOrder)) {
+ SPtrInfo.StrideSCEV = SE->getNegativeSCEV(SPtrInfo.StrideSCEV);
+ CurrentOrder.clear();
+ }
+
+ return TreeEntry::StridedVectorize;
+ }
LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
return TreeEntry::NeedToGather;
@@ -12946,9 +12975,11 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
BlockScheduling &BS = *BSRef;
+ bool NoInternalAlias =
+ isa<StoreInst>(VL0) && State == TreeEntry::StridedVectorize;
SetVector<Value *> UniqueValues(llvm::from_range, VL);
- std::optional<ScheduleBundle *> BundlePtr =
- BS.tryScheduleBundle(UniqueValues.getArrayRef(), this, S, UserTreeIdx);
+ std::optional<ScheduleBundle *> BundlePtr = BS.tryScheduleBundle(
+ UniqueValues.getArrayRef(), this, S, UserTreeIdx, NoInternalAlias);
#ifdef EXPENSIVE_CHECKS
// Make sure we didn't break any internal invariants
BS.verify();
@@ -23621,7 +23652,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
unsigned StridedStoreEC = getNumElements(StridedStoreTy);
Value *Stride = SPtrInfo.StrideVal;
- assert(Stride && "Missing StridedPointerInfo for tree entry.");
+ if (!Stride) {
+ const SCEV *StrideSCEV = SPtrInfo.StrideSCEV;
+ assert(StrideSCEV && "Neither StrideVal nor StrideSCEV were set.");
+ SCEVExpander Expander(*SE, "strided-load-vec");
+ Stride = Expander.expandCodeFor(StrideSCEV, StrideSCEV->getType(),
+ &*Builder.GetInsertPoint());
+ }
Value *StrideVal =
Builder.CreateIntCast(Stride, StrideTy, /*isSigned=*/true);
// vp_strided_store::stride is defined in bytes
@@ -25009,9 +25046,10 @@ void BoUpSLP::optimizeGatherSequence() {
}
BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
- ArrayRef<Value *> VL, const InstructionsState &S, const EdgeInfo &EI) {
- auto &BundlePtr =
- ScheduledBundlesList.emplace_back(std::make_unique<ScheduleBundle>());
+ ArrayRef<Value *> VL, const InstructionsState &S, const EdgeInfo &EI,
+ bool NoInternalAlias) {
+ auto &BundlePtr = ScheduledBundlesList.emplace_back(
+ std::make_unique<ScheduleBundle>(/*IsValid*/ true, NoInternalAlias));
for (Value *V : VL) {
if (S.isNonSchedulable(V))
continue;
@@ -25041,7 +25079,8 @@ BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
std::optional<BoUpSLP::ScheduleBundle *>
BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S,
- const EdgeInfo &EI) {
+ const EdgeInfo &EI,
+ bool NoInternalAlias) {
// No need to schedule PHIs, insertelement, extractelement and extractvalue
// instructions.
if (isa<PHINode>(S.getMainOp()) ||
@@ -25409,7 +25448,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
ReSchedule = true;
}
- ScheduleBundle &Bundle = buildBundle(VL, S, EI);
+ ScheduleBundle &Bundle = buildBundle(VL, S, EI, NoInternalAlias);
TryScheduleBundleImpl(ReSchedule, Bundle);
if (!Bundle.isReady()) {
for (ScheduleEntity *BD : Bundle.getBundle()) {
@@ -25874,15 +25913,18 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
(IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
SLP->isAliased(SrcLoc, SrcInst, DepDest->getInst())))) {
- // We increment the counter only if the locations are aliased
- // (instead of counting all alias checks). This gives a better
- // balance between reduced runtime and accurate dependencies.
- NumAliased++;
+ if (!Bundle.NoInternalAlias || !Bundle.findInst(DepDest->getInst())) {
+
+ // We increment the counter only if the locations are aliased
+ // (instead of counting all alias checks). This gives a better
+ // balance between reduced runtime and accurate dependencies.
+ NumAliased++;
- DepDest->addMemoryDependency(BundleMember);
- BundleMember->incDependencies();
- if (!DepDest->isScheduled())
- BundleMember->incrementUnscheduledDeps(1);
+ DepDest->addMemoryDependency(BundleMember);
+ BundleMember->incDependencies();
+ if (!DepDest->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
+ }
if (!DepDest->hasValidDependencies() ||
(InsertInReadyList && DepDest->isReady()))
WorkList.push_back(DepDest);
@@ -27265,8 +27307,21 @@ class StoreChainContext {
SmallVector<unsigned> &RangeSizesByIdx,
unsigned Stride)
: Operands(Ops), RangeSizesStorage(RangeSizes),
- RangeSizesByIdx(RangeSizesByIdx), Stride(Stride) {}
+ RangeSizesByIdx(RangeSizesByIdx), Stride(Stride), RTStride(nullptr) {}
+ explicit StoreChainContext(ArrayRef<Value *> Ops,
+ ArrayRef<SizePair> RangeSizes,
+ SmallVector<unsigned> &RangeSizesByIdx,
+ const SCEV *RTStride)
+ : Operands(Ops), RangeSizesStorage(RangeSizes),
+ RangeSizesByIdx(RangeSizesByIdx), Stride(0), RTStride(RTStride) {}
+
+ /// Compare two Contexts prioritize small constant strides
+ bool operator<(const StoreChainContext &Other) const {
+ if (RTStride || Other.RTStride)
+ return !RTStride;
+ return Stride < Other.Stride;
+ }
/// Set up initial values using the already set Operands
bool initializeContext(
BoUpSLP &R, const DataLayout &DL, const TargetTransformInfo &TTI,
@@ -27276,8 +27331,6 @@ class StoreChainContext {
std::optional<unsigned> getCurrentVF() const;
/// Return the maximum VF for the context
unsigned getMaxVF() const { return MaxVF; }
- /// Return the stride of the context
- unsigned getStride() const { return Stride; }
/// Attempt to vectorize Operands for the given VF
/// Returns false if no more attempts should be made for the context
bool vectorizeOneVF(const TargetTransformInfo &TTI, unsigned VF,
@@ -27387,6 +27440,8 @@ class StoreChainContext {
bool RepeatChanged = false;
/// For constant strided stores, what is the stride amount
const unsigned Stride = 0;
+ /// For runtime strided stores
+ const SCEV *RTStride = nullptr;
/// Store information about failed vectorization attempts due to scheduling
SmallDenseMap<Value *, SizePair> NonSchedulable;
};
@@ -27460,7 +27515,7 @@ bool StoreChainContext::initializeContext(
Store->getPointerAddressSpace()));
MinVF /= getNumElements(StoreTy);
MinVF = std::max<unsigned>(2, MinVF);
- if (Stride > 1)
+ if (Stride > 1 || RTStride)
MinVF = std::max<unsigned>(MinVF, MinProfitableStridedStores);
if (MaxVF < MinVF) {
@@ -27675,7 +27730,7 @@ bool StoreChainContext::vectorizeOneVF(
markRangeVectorized(SliceStartIdx, VF, FirstUnvecStore, MaxSliceEnd);
SliceStartIdx += VF;
++NumStoreChains;
- if (Stride > 1)
+ if (Stride > 1 || RTStride)
++NumStridedStoreChains;
NumVectorizedStores += VF;
continue;
@@ -27904,7 +27959,7 @@ bool SLPVectorizerPass::vectorizeStores(
llvm::stable_sort(AllContexts,
[](const std::unique_ptr<StoreChainContext> &A,
const std::unique_ptr<StoreChainContext> &B) {
- return A && (!B || A->getStride() < B->getStride());
+ return A && (!B || *A < *B);
});
for (unsigned LimitVF = GlobalMaxVF; LimitVF > 0;
@@ -28010,6 +28065,46 @@ bool SLPVectorizerPass::vectorizeStores(
for (RelatedStoreInsts &StoreSeq : SortedStores)
ExtendContexts(StoreSeq.getStores());
+ if (EnableStridedStores) {
+ // Stores is already reversed, reverse again so that we generate
+ // chains in program order in order for scheduling to occur
+ const unsigned NumStores = Stores.size();
+ unsigned Idx = NumStores - 1;
+ while (Idx > 0) {
+ StoreInst *StartSI = Stores[Idx];
+ StoreInst *NextSI = Stores[Idx - 1];
+ Type *StoreTy = StartSI->getValueOperand()->getType();
+ if (NextSI->getValueOperand()->getType() != StoreTy) {
+ --Idx;
+ continue;
+ }
+ const SCEV *StartSCEV = SE->getSCEV(StartSI->getPointerOperand());
+ const SCEV *NextSCEV = SE->getSCEV(NextSI->getPointerOperand());
+ const SCEV *Stride = SE->getMinusSCEV(NextSCEV, StartSCEV);
+ if (isa<SCEVCouldNotCompute, SCEVConstant, SCEVVScale>(Stride)) {
+ --Idx;
+ continue;
+ }
+ SmallVector<Value *> Ops{StartSI, NextSI};
+ SmallVector<StoreChainContext::SizePair> RS{{Idx, 1}, {Idx - 1, 1}};
+ --Idx;
+ while (Idx > 0) {
+ NextSI = Stores[Idx - 1];
+ if (NextSI->getValueOperand()->getType() != StoreTy)
+ break;
+ NextSCEV = SE->getAddExpr(NextSCEV, Stride);
+ if (NextSCEV != SE->getSCEV(NextSI->getPointerOperand()))
+ break;
+ --Idx;
+ Ops.push_back(NextSI);
+ RS.emplace_back(Idx, 1);
+ }
+ if (Ops.size() >= MinProfitableStridedStores)
+ AllContexts.emplace_back(std::make_unique<StoreChainContext>(
+ Ops, RS, RangeSizesByIdx, Stride));
+ }
+ }
+
ActuallyVectorizeContexts();
return Changed;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
index 0d5b1271bc86e..1e2391d0aa5d3 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
@@ -6,31 +6,10 @@ define void @runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i8.p0.i64(<8 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 [[TMP2]], <8 x i1> splat (i1 true), i32 8)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -76,39 +55,11 @@ define void @runtime_stride_complex_scev(ptr %pl, ptr %ps, i64 %stride, i64 %str
; CHECK-LABEL: define void @runtime_stride_complex_scev(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]], i64 [[STRIDE1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_SS0:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS1:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS2:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS3:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS4:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS5:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS6:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_SS7:%.*]] = getelementptr i8, ptr [[GEP_S7]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_SS0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_SS1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_SS2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_SS3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_SS4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_SS5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_SS6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_SS7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i8.p0.i64(<8 x i8> [[TMP1]], ptr align 1 [[GEP_SS0]], i64 [[TMP2]], <8 x i1> splat (i1 true), i32 8)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -164,32 +115,16 @@ define void @two_runtime_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @two_runtime_strides(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
; CHECK-NEXT: [[PS1:%.*]] = mul i64 [[STRIDE]], 16
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 [[PS1]]
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 [[TMP2]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 [[TMP4]], <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -236,31 +171,15 @@ define void @runtime_strides_constant_gap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_strides_constant_gap(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 100
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 [[TMP2]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 [[TMP4]], <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -306,33 +225,16 @@ define void @overlapping_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlapping_strides(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
-; CHECK-NEXT: [[STRIDE2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
; CHECK-NEXT: [[STRIDE1:%.*]] = mul i64 [[STRIDE]], 2
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE2]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE2]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE2]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[PS]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE2]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE2]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE2]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 [[TMP2]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 [[TMP4]], <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -380,9 +282,9 @@ define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_unit_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[GEP_L0]], <8 x i1> <i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_L2]], i64 5, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -392,14 +294,13 @@ define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 1
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[LOAD2:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP3]], ptr align 1 [[GEP_S0]], i64 [[TMP4]], <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[LOAD2:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[LOAD7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
+; CHECK-NEXT: [[LOAD7:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -447,25 +348,16 @@ define void @unit_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 1
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 1
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 1
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP2]], ptr align 1 [[GEP_S4]], i64 [[TMP3]], <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -512,21 +404,15 @@ define void @runtime_stride_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP2]], ptr align 1 [[GEP_S0]], i64 [[TMP4]], <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 2, <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
;
@@ -574,25 +460,16 @@ define void @constant_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 2
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 2
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 2
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 2, <4 x i1> splat (i1 true), i32 4)
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP2]], ptr align 1 [[GEP_S4]], i64 [[TMP3]], <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -640,17 +517,15 @@ define void @overlap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP2]], ptr align 1 [[GEP_S0]], i64 [[TMP3]], <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
; CHECK-NEXT: ret void
@@ -696,17 +571,15 @@ define void @overlap_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP2]], ptr align 1 [[GEP_S0]], i64 [[TMP3]], <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S3]], i64 2, <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
@@ -750,31 +623,11 @@ define void @runtime_stride_reorder(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_reorder(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 1, i32 4, i32 7, i32 5, i32 6, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v8i8.p0.i64(<8 x i8> [[TMP2]], ptr align 1 [[GEP_S0]], i64 [[TMP3]], <8 x i1> splat (i1 true), i32 8)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
@@ -820,7 +673,9 @@ define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_unschedulable(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[GEP_L0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true>, <8 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -829,21 +684,16 @@ define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: store i8 [[TMP10]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: store i8 [[TMP11]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S2]], i64 [[TMP6]], <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -919,37 +769,33 @@ define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP1:%.*]] = call <5 x i8> @llvm.masked.load.v5i8.p0(ptr align 1 [[GEP_L0]], <5 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true>, <5 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <5 x i8> [[TMP1]], <5 x i8> poison, <4 x i32> <i32 0, i32 2, i32 3, i32 4>
-; CHECK-NEXT: [[LOAD6:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_L0]], i64 4, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_L1]], i64 4, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
-; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
-; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
+; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
+; CHECK-NEXT: [[GEP_S8:%.*]] = getelementptr i8, ptr [[GEP_S7]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x i8>, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = load <2 x i8>, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0
; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i16
-; CHECK-NEXT: store i16 [[TMP6]], ptr [[GEP_S1]], align 2
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: store i16 [[TMP6]], ptr [[GEP_S3]], align 2
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP7]], ptr align 1 [[GEP_S4]], i64 [[TMP8]], <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S6]], align 1
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[TMP10]] to i16
-; CHECK-NEXT: store i16 [[TMP11]], ptr [[GEP_S5]], align 2
-; CHECK-NEXT: store i8 [[LOAD6]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
+; CHECK-NEXT: store i16 [[TMP11]], ptr [[GEP_S7]], align 2
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> [[TMP13]], ptr align 1 [[GEP_S8]], i64 [[TMP12]], <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: ret void
;
%gep_l0 = getelementptr i8, ptr %pl, i64 0
>From c6e18e0279bc85ed824253e09e9b47fe62f0a5b8 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 3 Jun 2026 10:58:15 -0700
Subject: [PATCH 03/14] Remove whitespace
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1751aaad2a6d0..c4df1d7491183 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -25941,7 +25941,6 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
SLP->isAliased(SrcLoc, SrcInst, DepDest->getInst())))) {
if (!Bundle.NoInternalAlias || !Bundle.findInst(DepDest->getInst())) {
-
// We increment the counter only if the locations are aliased
// (instead of counting all alias checks). This gives a better
// balance between reduced runtime and accurate dependencies.
>From 5e533b5c46e521cf02d1a06f04bbe51739af0c3f Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 3 Jun 2026 10:59:10 -0700
Subject: [PATCH 04/14] strided-load-vec -> strided-store-vec
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c4df1d7491183..46c0467edeef4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23682,7 +23682,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if (!Stride) {
const SCEV *StrideSCEV = SPtrInfo.StrideSCEV;
assert(StrideSCEV && "Neither StrideVal nor StrideSCEV were set.");
- SCEVExpander Expander(*SE, "strided-load-vec");
+ SCEVExpander Expander(*SE, "strided-store-vec");
Stride = Expander.expandCodeFor(StrideSCEV, StrideSCEV->getType(),
&*Builder.GetInsertPoint());
}
>From d64c33daa8604ad17343763e0518d626fe132da3 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 3 Jun 2026 11:09:16 -0700
Subject: [PATCH 05/14] Add assertion that at least one store is preset in
Stores
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 46c0467edeef4..c3ad6505b4db1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28112,6 +28112,7 @@ bool SLPVectorizerPass::vectorizeStores(
// Stores is already reversed, reverse again so that we generate
// chains in program order in order for scheduling to occur
const unsigned NumStores = Stores.size();
+ assert(NumStores > 0 && "Expected at least one StoreInst");
unsigned Idx = NumStores - 1;
while (Idx > 0) {
StoreInst *StartSI = Stores[Idx];
>From 85e8cd45aea7a2ca3eee8ada7d92104a3c733018 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 3 Jun 2026 11:20:45 -0700
Subject: [PATCH 06/14] Check for EnableStridedStores when checking
analyzeRtStrideCandidate()
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c3ad6505b4db1..0200969e9fbc1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11075,8 +11075,8 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
CurrentOrder, *Dist, Ptr0, SPtrInfo))
return TreeEntry::StridedVectorize;
}
- if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
- CurrentOrder, SPtrInfo, /*isLoad=*/false)) {
+ if (EnableStridedStores && analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
+ CurrentOrder, SPtrInfo, /*isLoad=*/false)) {
if (isIdentityOrder(CurrentOrder)) {
CurrentOrder.clear();
} else if (isReverseOrder(CurrentOrder)) {
>From 2e1f538e01fe237e1e04fbc8625b2bf5de797258 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 3 Jun 2026 11:40:37 -0700
Subject: [PATCH 07/14] Only compute CommonAlignment when required
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 ++++++++++---------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0200969e9fbc1..11a6086cf73b9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11052,8 +11052,6 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
PointerOps.push_back(SI->getPointerOperand());
}
- Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
-
// Check the order of pointer operands.
if (llvm::sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, CurrentOrder)) {
Value *Ptr0;
@@ -11070,21 +11068,24 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
// Check that the sorted pointer operands are consecutive.
if (static_cast<uint64_t>(*Dist) == VL.size() - 1)
return TreeEntry::Vectorize;
+ Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
if (EnableStridedStores &&
analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
CurrentOrder, *Dist, Ptr0, SPtrInfo))
return TreeEntry::StridedVectorize;
}
- if (EnableStridedStores && analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
- CurrentOrder, SPtrInfo, /*isLoad=*/false)) {
- if (isIdentityOrder(CurrentOrder)) {
- CurrentOrder.clear();
- } else if (isReverseOrder(CurrentOrder)) {
- SPtrInfo.StrideSCEV = SE->getNegativeSCEV(SPtrInfo.StrideSCEV);
- CurrentOrder.clear();
+ if (EnableStridedStores) {
+ Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
+ if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
+ CurrentOrder, SPtrInfo, /*isLoad=*/false)) {
+ if (isIdentityOrder(CurrentOrder)) {
+ CurrentOrder.clear();
+ } else if (isReverseOrder(CurrentOrder)) {
+ SPtrInfo.StrideSCEV = SE->getNegativeSCEV(SPtrInfo.StrideSCEV);
+ CurrentOrder.clear();
+ }
+ return TreeEntry::StridedVectorize;
}
-
- return TreeEntry::StridedVectorize;
}
LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
>From ec8a9a2af19317510a7e2b70be101670385841a7 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 3 Jun 2026 12:53:24 -0700
Subject: [PATCH 08/14] Don't detect 0-stride cases in
analyzeRtStrideCandidate()
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 11a6086cf73b9..054f963e80f60 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7705,7 +7705,7 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
SmallVector<unsigned> SortedIndicesForOffset0;
const SCEV *Stride0 = calculateRtStride(PointerOps0, BaseTy, *DL, *SE,
SortedIndicesForOffset0, Coeffs0);
- if (!Stride0)
+ if (!Stride0 || isa<SCEVConstant>(Stride0))
return false;
unsigned NumCoeffs0 = Coeffs0.size();
if (NumCoeffs0 * NumOffsets != Sz)
>From abf979c2dfcf55a331524ea6e5d4ebc88d3b9da4 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Mon, 22 Jun 2026 12:28:05 -0700
Subject: [PATCH 09/14] [SLP] Remove 'NoInternalAlias'
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 58 ++++++-------------
1 file changed, 18 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 054f963e80f60..69ca3a906389c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5347,15 +5347,10 @@ class slpvectorizer::BoUpSLP {
public:
ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
- ScheduleBundle(bool IsValid, bool NoInternalAlias)
- : ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid),
- NoInternalAlias(NoInternalAlias) {}
static bool classof(const ScheduleEntity *Entity) {
return Entity->getKind() == Kind::ScheduleBundle;
}
- bool NoInternalAlias = false;
-
/// Verify basic self consistency properties
void verify() const {
for (const ScheduleEntity *SD : Bundle) {
@@ -5374,14 +5369,6 @@ class slpvectorizer::BoUpSLP {
}
}
- /// Check to see if an instruction is present in this bundle
- bool findInst(Instruction *I) {
- return std::find_if(Bundle.begin(), Bundle.end(),
- [&](ScheduleEntity *SE) {
- return SE->getInst() == I;
- }) != Bundle.end();
- }
-
/// Returns the number of unscheduled dependencies in the bundle.
int unscheduledDepsInBundle() const {
assert(*this && "bundle must not be empty");
@@ -6272,11 +6259,8 @@ class slpvectorizer::BoUpSLP {
/// \param VL The list of scalar instructions.
/// \param S The state of the instructions.
/// \param EI The edge in the SLP graph or the user node/operand number.
- /// \param NoInternalAlias Ignore any schedule issues due to aliasing within
- /// the bundle.
ScheduleBundle &buildBundle(ArrayRef<Value *> VL,
- const InstructionsState &S, const EdgeInfo &EI,
- bool NoInternalAlias);
+ const InstructionsState &S, const EdgeInfo &EI);
/// Checks if a bundle of instructions can be scheduled, i.e. has no
/// cyclic dependencies. This is only a dry-run, no instructions are
@@ -6285,8 +6269,7 @@ class slpvectorizer::BoUpSLP {
/// std::nullopt if \p VL is allowed to be scheduled.
std::optional<ScheduleBundle *>
tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
- const InstructionsState &S, const EdgeInfo &EI,
- bool NoInternalAlias);
+ const InstructionsState &S, const EdgeInfo &EI);
/// Allocates schedule data chunk.
ScheduleData *allocateScheduleDataChunks();
@@ -12976,11 +12959,9 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
BlockScheduling &BS = *BSRef;
- bool NoInternalAlias =
- isa<StoreInst>(VL0) && State == TreeEntry::StridedVectorize;
SetVector<Value *> UniqueValues(llvm::from_range, VL);
- std::optional<ScheduleBundle *> BundlePtr = BS.tryScheduleBundle(
- UniqueValues.getArrayRef(), this, S, UserTreeIdx, NoInternalAlias);
+ std::optional<ScheduleBundle *> BundlePtr =
+ BS.tryScheduleBundle(UniqueValues.getArrayRef(), this, S, UserTreeIdx);
#ifdef EXPENSIVE_CHECKS
// Make sure we didn't break any internal invariants
BS.verify();
@@ -25074,10 +25055,9 @@ void BoUpSLP::optimizeGatherSequence() {
}
BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
- ArrayRef<Value *> VL, const InstructionsState &S, const EdgeInfo &EI,
- bool NoInternalAlias) {
- auto &BundlePtr = ScheduledBundlesList.emplace_back(
- std::make_unique<ScheduleBundle>(/*IsValid*/ true, NoInternalAlias));
+ ArrayRef<Value *> VL, const InstructionsState &S, const EdgeInfo &EI) {
+ auto &BundlePtr =
+ ScheduledBundlesList.emplace_back(std::make_unique<ScheduleBundle>());
for (Value *V : VL) {
if (S.isNonSchedulable(V))
continue;
@@ -25107,8 +25087,7 @@ BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
std::optional<BoUpSLP::ScheduleBundle *>
BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S,
- const EdgeInfo &EI,
- bool NoInternalAlias) {
+ const EdgeInfo &EI) {
// No need to schedule PHIs, insertelement, extractelement and extractvalue
// instructions.
if (isa<PHINode>(S.getMainOp()) ||
@@ -25476,7 +25455,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
ReSchedule = true;
}
- ScheduleBundle &Bundle = buildBundle(VL, S, EI, NoInternalAlias);
+ ScheduleBundle &Bundle = buildBundle(VL, S, EI);
TryScheduleBundleImpl(ReSchedule, Bundle);
if (!Bundle.isReady()) {
for (ScheduleEntity *BD : Bundle.getBundle()) {
@@ -25941,17 +25920,16 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
(IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
SLP->isAliased(SrcLoc, SrcInst, DepDest->getInst())))) {
- if (!Bundle.NoInternalAlias || !Bundle.findInst(DepDest->getInst())) {
- // We increment the counter only if the locations are aliased
- // (instead of counting all alias checks). This gives a better
- // balance between reduced runtime and accurate dependencies.
- NumAliased++;
+ // We increment the counter only if the locations are aliased
+ // (instead of counting all alias checks). This gives a better
+ // balance between reduced runtime and accurate dependencies.
+ NumAliased++;
+
+ DepDest->addMemoryDependency(BundleMember);
+ BundleMember->incDependencies();
+ if (!DepDest->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
- DepDest->addMemoryDependency(BundleMember);
- BundleMember->incDependencies();
- if (!DepDest->isScheduled())
- BundleMember->incrementUnscheduledDeps(1);
- }
if (!DepDest->hasValidDependencies() ||
(InsertInReadyList && DepDest->isReady()))
WorkList.push_back(DepDest);
>From c34feff09083f3ff334d04d05276c681b1675abd Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Wed, 24 Jun 2026 11:44:41 -0700
Subject: [PATCH 10/14] [SLP] Fix scheduling to allow may-alias stores to be in
the same Bundle
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 105 +++++++++++++++++-
1 file changed, 104 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 69ca3a906389c..c23cf8ad36a7b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6181,6 +6181,8 @@ class slpvectorizer::BoUpSLP {
});
};
for (ScheduleEntity *SD : Bundle.getBundle()) {
+ if (SD->isScheduled())
+ continue;
ArrayRef<ScheduleBundle *> SDBundles;
if (!isa<ScheduleCopyableData>(SD))
SDBundles = getScheduleBundles(SD->getInst());
@@ -25385,10 +25387,51 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
initialFillReadyList(ReadyInsts);
}
+ auto ScheduleInternalMembers = [&]() -> void {
+ if (Bundle.isReady())
+ return;
+ SmallDenseMap<ScheduleEntity *, std::pair<unsigned, int>> BundleDeps;
+ for (auto [Idx, BM] : enumerate(Bundle.getBundle()))
+ BundleDeps[BM] = {Idx, BM->getUnscheduledDeps()};
+ for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
+ if (BundleMember->isScheduled())
+ continue;
+ auto *SD = dyn_cast<ScheduleData>(BundleMember);
+ if (!SD || !isa<StoreInst>(SD->getInst()))
+ return;
+ for (ScheduleData *MemDep : SD->getMemoryDependencies())
+ if (BundleDeps.contains(MemDep))
+ if (BundleDeps[MemDep].first < BundleDeps[SD].first)
+ --BundleDeps[MemDep].second;
+ }
+ for (auto [_, Cnt] : BundleDeps)
+ if (Cnt.second)
+ return;
+
+ bool LocalProgress = false;
+ do {
+ LocalProgress = false;
+ for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
+ if (BundleMember->isScheduled() || !BundleMember->isReady())
+ continue;
+ LLVM_DEBUG(dbgs() << "SLP: schedule ready bundle member "
+ << *BundleMember << "\n");
+ ReadyInsts.remove(BundleMember);
+ schedule(*SLP, S, EI, BundleMember, ReadyInsts);
+ LocalProgress = true;
+ }
+ } while (!Bundle.isReady() && LocalProgress);
+ assert(Bundle.isReady() &&
+ "Bundle must be ready after internal scheduling");
+ };
+
// Now try to schedule the new bundle or (if no bundle) just calculate
// dependencies. As soon as the bundle is "ready" it means that there are no
// cyclic dependencies and we can schedule it. Note that's important that we
// don't "schedule" the bundle yet.
+ if (Bundle && !Bundle.isReady()) {
+ ScheduleInternalMembers();
+ }
while (((!Bundle && ReSchedule) || (Bundle && !Bundle.isReady())) &&
!ReadyInsts.empty()) {
ScheduleEntity *Picked = ReadyInsts.pop_back_val();
@@ -26140,9 +26183,69 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
BS->initialFillReadyList(ReadyInsts);
Instruction *LastScheduledInst = BS->ScheduleEnd;
+ SmallPtrSet<Instruction *, 16> Scheduled;
+
+ auto ScheduleInternalMembers = [&](ScheduleBundle &Bundle) -> void {
+ if (Bundle.isReady())
+ return;
+ SmallDenseMap<ScheduleEntity *, std::pair<unsigned, int>> BundleDeps;
+ for (auto [Idx, BM] : enumerate(Bundle.getBundle()))
+ BundleDeps[BM] = {Idx, BM->getUnscheduledDeps()};
+ for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
+ if (BundleMember->isScheduled())
+ continue;
+ auto *SD = dyn_cast<ScheduleData>(BundleMember);
+ if (!SD || !isa<StoreInst>(SD->getInst()))
+ return;
+ for (ScheduleData *MemDep : SD->getMemoryDependencies())
+ if (BundleDeps.contains(MemDep))
+ if (BundleDeps[MemDep].first < BundleDeps[SD].first)
+ --BundleDeps[MemDep].second;
+ }
+ for (auto [_, Cnt] : BundleDeps)
+ if (Cnt.second)
+ return;
+
+ bool LocalProgress = false;
+ do {
+ LocalProgress = false;
+ for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
+ if (BundleMember->isScheduled() || !BundleMember->isReady())
+ continue;
+ LLVM_DEBUG(dbgs() << "SLP: schedule ready bundle member "
+ << *BundleMember << "\n");
+ ReadyInsts.erase(BundleMember);
+ auto *SD = cast<ScheduleData>(BundleMember);
+ Instruction *PickedInst = SD->getInst();
+ if (PickedInst->getNextNode() != LastScheduledInst)
+ PickedInst->moveAfter(LastScheduledInst->getPrevNode());
+ LastScheduledInst = PickedInst;
+ auto Invalid = InstructionsState::invalid();
+ BS->schedule(R, Invalid, EdgeInfo(), BundleMember, ReadyInsts);
+ Scheduled.insert(PickedInst);
+
+ LocalProgress = true;
+ }
+ } while (!Bundle.isReady() && LocalProgress);
+ assert(Bundle.isReady() &&
+ "Bundle must be ready after internal scheduling");
+ };
+
+ for (std::unique_ptr<ScheduleBundle> &Bundle : BS->ScheduledBundlesList) {
+ if (!Bundle->hasValidDependencies() || Bundle->isReady())
+ continue;
+ if (all_of(Bundle->getBundle(),
+ [](const ScheduleEntity *SE) { return !SE->isReady(); }))
+ continue;
+ if (!all_of(Bundle->getBundle(), [](const ScheduleEntity *SE) {
+ auto *SD = dyn_cast<ScheduleData>(SE);
+ return SD && isa<StoreInst>(SD->getInst());
+ }))
+ continue;
+ ScheduleInternalMembers(*Bundle);
+ }
// Do the "real" scheduling.
- SmallPtrSet<Instruction *, 16> Scheduled;
while (!ReadyInsts.empty()) {
auto *Picked = *ReadyInsts.begin();
ReadyInsts.erase(ReadyInsts.begin());
>From 808a857c1b17b9008acd04846ca40c62007875ac Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 26 Jun 2026 15:19:11 -0700
Subject: [PATCH 11/14] [SLP] Better handle ordering as part of
analyzeRtStrideCandidate()
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 148 ++++++++----------
1 file changed, 61 insertions(+), 87 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c23cf8ad36a7b..1b020a0774820 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6256,6 +6256,50 @@ class slpvectorizer::BoUpSLP {
}
}
+ /// Handle may-alias stores within the same bundle, have to be scheduled one
+ /// at a time since later nodes are not ready until earlier stores are
+ /// scheduled. Must ensure that the ordering of scheduling matches order in
+ /// bundle
+ void
+ scheduleInternalStores(ScheduleBundle &Bundle,
+ function_ref<void(ScheduleEntity *)> ScheduleCall) {
+ if (Bundle.isReady())
+ return;
+ // Make sure all bundle members are:
+ // (1) Only dependent on other members of the bundle
+ // (2) Only dependent on members after them in the bundle
+ // (3) Store instructions
+ // If all of these are satisfied, then can schedule them
+ // one at a time
+ SmallDenseMap<ScheduleEntity *, unsigned> BundleMembers;
+ for (auto [Idx, BM] : enumerate(Bundle.getBundle()))
+ BundleMembers[BM] = Idx;
+ for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
+ auto *SD = dyn_cast<ScheduleData>(BundleMember);
+ if (!SD || !isa<StoreInst>(SD->getInst()))
+ return;
+ if (BundleMember->isScheduled())
+ continue;
+ for (ScheduleData *MemDep : SD->getMemoryDependencies()) {
+ if (!BundleMembers.contains(MemDep))
+ return;
+ if (BundleMembers[MemDep] >= BundleMembers[SD])
+ return;
+ }
+ }
+
+ while (!Bundle.isReady()) {
+ for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
+ if (BundleMember->isScheduled() || !BundleMember->isReady())
+ continue;
+ LLVM_DEBUG(dbgs() << "SLP: schedule ready bundle member "
+ << *BundleMember << "\n");
+ ScheduleCall(BundleMember);
+ }
+ }
+ return;
+ }
+
/// Build a bundle from the ScheduleData nodes corresponding to the
/// scalar instruction for each lane.
/// \param VL The list of scalar instructions.
@@ -25387,50 +25431,16 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
initialFillReadyList(ReadyInsts);
}
- auto ScheduleInternalMembers = [&]() -> void {
- if (Bundle.isReady())
- return;
- SmallDenseMap<ScheduleEntity *, std::pair<unsigned, int>> BundleDeps;
- for (auto [Idx, BM] : enumerate(Bundle.getBundle()))
- BundleDeps[BM] = {Idx, BM->getUnscheduledDeps()};
- for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
- if (BundleMember->isScheduled())
- continue;
- auto *SD = dyn_cast<ScheduleData>(BundleMember);
- if (!SD || !isa<StoreInst>(SD->getInst()))
- return;
- for (ScheduleData *MemDep : SD->getMemoryDependencies())
- if (BundleDeps.contains(MemDep))
- if (BundleDeps[MemDep].first < BundleDeps[SD].first)
- --BundleDeps[MemDep].second;
- }
- for (auto [_, Cnt] : BundleDeps)
- if (Cnt.second)
- return;
-
- bool LocalProgress = false;
- do {
- LocalProgress = false;
- for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
- if (BundleMember->isScheduled() || !BundleMember->isReady())
- continue;
- LLVM_DEBUG(dbgs() << "SLP: schedule ready bundle member "
- << *BundleMember << "\n");
- ReadyInsts.remove(BundleMember);
- schedule(*SLP, S, EI, BundleMember, ReadyInsts);
- LocalProgress = true;
- }
- } while (!Bundle.isReady() && LocalProgress);
- assert(Bundle.isReady() &&
- "Bundle must be ready after internal scheduling");
- };
-
// Now try to schedule the new bundle or (if no bundle) just calculate
// dependencies. As soon as the bundle is "ready" it means that there are no
// cyclic dependencies and we can schedule it. Note that's important that we
// don't "schedule" the bundle yet.
if (Bundle && !Bundle.isReady()) {
- ScheduleInternalMembers();
+ auto ScheduleCall = [&](ScheduleEntity *SE) -> void {
+ ReadyInsts.remove(SE);
+ schedule(*SLP, S, EI, SE, ReadyInsts);
+ };
+ scheduleInternalStores(Bundle, ScheduleCall);
}
while (((!Bundle && ReSchedule) || (Bundle && !Bundle.isReady())) &&
!ReadyInsts.empty()) {
@@ -26184,53 +26194,6 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
Instruction *LastScheduledInst = BS->ScheduleEnd;
SmallPtrSet<Instruction *, 16> Scheduled;
-
- auto ScheduleInternalMembers = [&](ScheduleBundle &Bundle) -> void {
- if (Bundle.isReady())
- return;
- SmallDenseMap<ScheduleEntity *, std::pair<unsigned, int>> BundleDeps;
- for (auto [Idx, BM] : enumerate(Bundle.getBundle()))
- BundleDeps[BM] = {Idx, BM->getUnscheduledDeps()};
- for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
- if (BundleMember->isScheduled())
- continue;
- auto *SD = dyn_cast<ScheduleData>(BundleMember);
- if (!SD || !isa<StoreInst>(SD->getInst()))
- return;
- for (ScheduleData *MemDep : SD->getMemoryDependencies())
- if (BundleDeps.contains(MemDep))
- if (BundleDeps[MemDep].first < BundleDeps[SD].first)
- --BundleDeps[MemDep].second;
- }
- for (auto [_, Cnt] : BundleDeps)
- if (Cnt.second)
- return;
-
- bool LocalProgress = false;
- do {
- LocalProgress = false;
- for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
- if (BundleMember->isScheduled() || !BundleMember->isReady())
- continue;
- LLVM_DEBUG(dbgs() << "SLP: schedule ready bundle member "
- << *BundleMember << "\n");
- ReadyInsts.erase(BundleMember);
- auto *SD = cast<ScheduleData>(BundleMember);
- Instruction *PickedInst = SD->getInst();
- if (PickedInst->getNextNode() != LastScheduledInst)
- PickedInst->moveAfter(LastScheduledInst->getPrevNode());
- LastScheduledInst = PickedInst;
- auto Invalid = InstructionsState::invalid();
- BS->schedule(R, Invalid, EdgeInfo(), BundleMember, ReadyInsts);
- Scheduled.insert(PickedInst);
-
- LocalProgress = true;
- }
- } while (!Bundle.isReady() && LocalProgress);
- assert(Bundle.isReady() &&
- "Bundle must be ready after internal scheduling");
- };
-
for (std::unique_ptr<ScheduleBundle> &Bundle : BS->ScheduledBundlesList) {
if (!Bundle->hasValidDependencies() || Bundle->isReady())
continue;
@@ -26242,7 +26205,18 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
return SD && isa<StoreInst>(SD->getInst());
}))
continue;
- ScheduleInternalMembers(*Bundle);
+ auto ScheduleCall = [&](ScheduleEntity *SE) -> void {
+ ReadyInsts.erase(SE);
+ auto *SD = cast<ScheduleData>(SE);
+ Instruction *PickedInst = SD->getInst();
+ if (PickedInst->getNextNode() != LastScheduledInst)
+ PickedInst->moveAfter(LastScheduledInst->getPrevNode());
+ LastScheduledInst = PickedInst;
+ auto Invalid = InstructionsState::invalid();
+ BS->schedule(R, Invalid, EdgeInfo(), SE, ReadyInsts);
+ Scheduled.insert(PickedInst);
+ };
+ BS->scheduleInternalStores(*Bundle, ScheduleCall);
}
// Do the "real" scheduling.
>From aeb7286860deb74ce2bf9e4eb6ab9d9fd45c823a Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Fri, 26 Jun 2026 15:19:39 -0700
Subject: [PATCH 12/14] [SLP] Fix handling of ordering around
analyzeRtStrideCandidate()
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 26 ++++++++++++++-----
1 file changed, 20 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1b020a0774820..7b7b730e81257 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7777,6 +7777,26 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
SortedIndices = std::move(SortedIndicesDraft);
SPtrInfo.StrideSCEV = Stride0;
SPtrInfo.Ty = StridedLoadTy;
+
+ auto isReverseWidenedOrder = [&](ArrayRef<unsigned> Order,
+ unsigned NumOffsets) -> bool {
+ if (Order.empty())
+ return false;
+ unsigned Sz = Order.size();
+ return all_of(enumerate(Order), [&](const auto &Pair) {
+ return Pair.value() == Sz ||
+ (NumOffsets - (Pair.index() / NumOffsets) - 1 ==
+ (Pair.value() / NumOffsets) &&
+ (Pair.index() % NumOffsets) == (Pair.value() % NumOffsets));
+ });
+ };
+
+ if (isIdentityOrder(SortedIndices)) {
+ SortedIndices.clear();
+ } else if (isReverseWidenedOrder(SortedIndices, NumOffsets)) {
+ SPtrInfo.StrideSCEV = SE->getNegativeSCEV(SPtrInfo.StrideSCEV);
+ SortedIndices.clear();
+ }
return true;
}
@@ -11107,12 +11127,6 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
CurrentOrder, SPtrInfo, /*isLoad=*/false)) {
- if (isIdentityOrder(CurrentOrder)) {
- CurrentOrder.clear();
- } else if (isReverseOrder(CurrentOrder)) {
- SPtrInfo.StrideSCEV = SE->getNegativeSCEV(SPtrInfo.StrideSCEV);
- CurrentOrder.clear();
- }
return TreeEntry::StridedVectorize;
}
}
>From 33cb74b9a2e6e26c20c76e5b4dd199a679d940a2 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Mon, 29 Jun 2026 13:54:58 -0700
Subject: [PATCH 13/14] [SLP] Ensure all bundle members are only dependent on
internal members
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7b7b730e81257..610fd299614aa 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6271,9 +6271,9 @@ class slpvectorizer::BoUpSLP {
// (3) Store instructions
// If all of these are satisfied, then can schedule them
// one at a time
- SmallDenseMap<ScheduleEntity *, unsigned> BundleMembers;
+ SmallDenseMap<ScheduleEntity *, std::pair<unsigned, int>> BundleMembers;
for (auto [Idx, BM] : enumerate(Bundle.getBundle()))
- BundleMembers[BM] = Idx;
+ BundleMembers[BM] = {Idx, BM->getUnscheduledDeps()};
for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
auto *SD = dyn_cast<ScheduleData>(BundleMember);
if (!SD || !isa<StoreInst>(SD->getInst()))
@@ -6282,19 +6282,26 @@ class slpvectorizer::BoUpSLP {
continue;
for (ScheduleData *MemDep : SD->getMemoryDependencies()) {
if (!BundleMembers.contains(MemDep))
+ continue;
+ if (BundleMembers[MemDep].first >= BundleMembers[SD].first)
return;
- if (BundleMembers[MemDep] >= BundleMembers[SD])
- return;
+ --BundleMembers[MemDep].second;
}
}
+ for (auto &[BM, P] : BundleMembers)
+ if (P.second != 0)
+ return;
- while (!Bundle.isReady()) {
+ bool ForwardProgress = true;
+ while (ForwardProgress) {
+ ForwardProgress = false;
for (ScheduleEntity *BundleMember : Bundle.getBundle()) {
if (BundleMember->isScheduled() || !BundleMember->isReady())
continue;
LLVM_DEBUG(dbgs() << "SLP: schedule ready bundle member "
<< *BundleMember << "\n");
ScheduleCall(BundleMember);
+ ForwardProgress = true;
}
}
return;
>From 7ea731a1f306cf660d1ab3fb4f0272e658038b77 Mon Sep 17 00:00:00 2001
From: bababuck <buchner.ryan at gmail.com>
Date: Mon, 29 Jun 2026 14:52:41 -0700
Subject: [PATCH 14/14] [SLP] Properly handle re-ordering
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 22 ++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 610fd299614aa..46375d2b655c2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6262,8 +6262,9 @@ class slpvectorizer::BoUpSLP {
/// bundle
void
scheduleInternalStores(ScheduleBundle &Bundle,
- function_ref<void(ScheduleEntity *)> ScheduleCall) {
- if (Bundle.isReady())
+ function_ref<void(ScheduleEntity *)> ScheduleCall,
+ const OrdersType &CurrentOrder) {
+ if (Bundle.isReady() || !CurrentOrder.empty())
return;
// Make sure all bundle members are:
// (1) Only dependent on other members of the bundle
@@ -6322,7 +6323,8 @@ class slpvectorizer::BoUpSLP {
/// std::nullopt if \p VL is allowed to be scheduled.
std::optional<ScheduleBundle *>
tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
- const InstructionsState &S, const EdgeInfo &EI);
+ const InstructionsState &S, const EdgeInfo &EI,
+ const OrdersType &CurrentOrder);
/// Allocates schedule data chunk.
ScheduleData *allocateScheduleDataChunks();
@@ -13027,8 +13029,8 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
BlockScheduling &BS = *BSRef;
SetVector<Value *> UniqueValues(llvm::from_range, VL);
- std::optional<ScheduleBundle *> BundlePtr =
- BS.tryScheduleBundle(UniqueValues.getArrayRef(), this, S, UserTreeIdx);
+ std::optional<ScheduleBundle *> BundlePtr = BS.tryScheduleBundle(
+ UniqueValues.getArrayRef(), this, S, UserTreeIdx, CurrentOrder);
#ifdef EXPENSIVE_CHECKS
// Make sure we didn't break any internal invariants
BS.verify();
@@ -25154,7 +25156,8 @@ BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
std::optional<BoUpSLP::ScheduleBundle *>
BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S,
- const EdgeInfo &EI) {
+ const EdgeInfo &EI,
+ const OrdersType &CurrentOrder) {
// No need to schedule PHIs, insertelement, extractelement and extractvalue
// instructions.
if (isa<PHINode>(S.getMainOp()) ||
@@ -25461,7 +25464,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
ReadyInsts.remove(SE);
schedule(*SLP, S, EI, SE, ReadyInsts);
};
- scheduleInternalStores(Bundle, ScheduleCall);
+ scheduleInternalStores(Bundle, ScheduleCall, CurrentOrder);
}
while (((!Bundle && ReSchedule) || (Bundle && !Bundle.isReady())) &&
!ReadyInsts.empty()) {
@@ -26237,7 +26240,10 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
BS->schedule(R, Invalid, EdgeInfo(), SE, ReadyInsts);
Scheduled.insert(PickedInst);
};
- BS->scheduleInternalStores(*Bundle, ScheduleCall);
+ // Order already checked when scheduling initially. The TreeEntry's order
+ // now may reflect ordering pushed down the tree.
+ OrdersType EmptyOrder;
+ BS->scheduleInternalStores(*Bundle, ScheduleCall, EmptyOrder);
}
// Do the "real" scheduling.
More information about the llvm-commits
mailing list