[llvm] [VPlan] Implement VPlan-based unit-strideness speculation (PR #182595)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 16:58:14 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/182595
>From bb7eed2fe0ef87e9a41f104552297e579d424138 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 20 Feb 2026 11:33:45 -0800
Subject: [PATCH 01/13] [NFC][VPlan] Add initial tests for future VPlan-based
stride MV
I tried to include both the features that current
LoopAccessAnalysis-based transformation supports (e.g., trunc/sext of
stride) but also cases where the current implementation behaves poorly,
e.g., https://godbolt.org/z/h31c3zKxK; as well as some other potentially
interesting scenarios I could imagine.
The are two test files with the same content. One is for VPlan dump change of
the future transformation alone (I'll update `-vplan-print-after` in the next
PR), another is for the full vectorizer pipeline. The latter have two `RUN:`
lines:
* No multiversioning, so the next PR diff can show the transformation itself
* Stride multiversionin performed in LAA, so that we can compare future
VPlan-based transformation vs old behavior.
---
.../VPlan/vplan-based-stride-mv.ll | 3417 ++++++++++++
.../LoopVectorize/vplan-based-stride-mv.ll | 4731 +++++++++++++++++
2 files changed, 8148 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
new file mode 100644
index 0000000000000..09f4b71594185
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -0,0 +1,3417 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -disable-output \
+; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
+; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s
+
+define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'basic'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
+; CHECK-LABEL: VPlan for loop in 'basic_optsize'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+attributes #0 = { optsize }
+
+define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
+; CHECK-LABEL: VPlan for loop in 'basic_minsize'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+attributes #1 = { minsize }
+
+; When using byte-gep with wide memop unit-stride wouldn't be one at the geps
+; index but rather mem-access-type-size. This test has constant multiplier equal
+; to that exact value `8 * %stride` so that `%stride == 1` would result in
+; unit-strided load.
+define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_scaled_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<8>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride.x8 = mul i64 %stride, 8
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride.x8
+; CHECK-NEXT: IR %gep.ld = getelementptr i8, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 8
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above but constant multiplier is smaller than load type's width, so
+; unit-strideness would require `%stride == 2`.
+define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_under_scaled_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<4>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride.x8 = mul i64 %stride, 4
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride.x8
+; CHECK-NEXT: IR %gep.ld = getelementptr i8, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 4
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Another variation for constant multiplier with byte gep. This time the
+; multiplier is bigger than load access type so this cannot be speculated for
+; unit-strideness.
+define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_over_scaled_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<16>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride.x8 = mul i64 %stride, 16
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride.x8
+; CHECK-NEXT: IR %gep.ld = getelementptr i8, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 16
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; And another one, with multiplier non-power-of-two. Cannot be speculated for
+; unit-strideness.
+define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_non_power_of_two_scaled_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<11>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride.x8 = mul i64 %stride, 11
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride.x8
+; CHECK-NEXT: IR %gep.ld = getelementptr i8, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 11
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; No constant multiplier, need to speculate `%stride == sizeof(load-access-type)`.
+define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_nonscaled_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i8, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; If we negate %stride before indexing, this might be a good heuristic to avoid
+; stride speculation... At the very least, keep this test for the documentation
+; purposes.
+define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_negated_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride.neg> = sub ir<0>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.neg>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride.neg = sub i64 0, %stride
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride.neg
+; CHECK-NEXT: IR %gep.ld = getelementptr i8, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.neg = sub i64 0, %stride
+ %idx = mul i64 %iv, %stride.neg
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Two memory accesses can be speculated for unit-strideness by single predicate.
+define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'shared_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %ld1 = load i64, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Speculating one access for unit-strideness guarantees that the other one isn't.
+define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'dependent_strides'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride1> = add ir<%stride>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride1 = add i64 %stride, 1
+; CHECK-NEXT: IR %idx0 = mul i64 %iv, %stride
+; CHECK-NEXT: IR %idx1 = mul i64 %iv, %stride1
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %ld1 = load i64, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride1 = add i64 %stride, 1
+ %idx0 = mul i64 %iv, %stride
+ %idx1 = mul i64 %iv, %stride1
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Same as above but in different order. Might be reasonable to have some
+; heuristic to choose one over another instead of just speculating the first
+; access.
+define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'dependent_strides_reverse_order'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride0> = add ir<%stride>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %stride0 = add i64 %stride, 1
+; CHECK-NEXT: IR %idx0 = mul i64 %iv, %stride0
+; CHECK-NEXT: IR %idx1 = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %ld1 = load i64, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride0 = add i64 %stride, 1
+ %idx0 = mul i64 %iv, %stride0
+ %idx1 = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Two dependent accesses again, but instead of strides being dependent on each
+; other, we have the same stride but different access sizes.
+define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_dependent_byte_geps'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1.ext> = sext ir<%ld1> to i64
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1.ext>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i8, ptr %p0, i64 %idx
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i8, ptr %p1, i64 %idx
+; CHECK-NEXT: IR %ld1 = load i32, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %ld1.ext = sext i32 %ld1 to i64
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1.ext
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i8, ptr %p0, i64 %idx
+ %ld0 = load i64, ptr %gep.ld0, align 8
+
+ %gep.ld1 = getelementptr i8, ptr %p1, i64 %idx
+ %ld1 = load i32, ptr %gep.ld1, align 8
+ %ld1.ext = sext i32 %ld1 to i64
+
+ %val = add i64 %ld0, %ld1.ext
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Likewise but reverse order of accesses.
+define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_dependent_byte_geps_reverse_order'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1.ext> = sext ir<%ld1> to i64
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1.ext>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i8, ptr %p1, i64 %idx
+; CHECK-NEXT: IR %ld1 = load i32, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %ld1.ext = sext i32 %ld1 to i64
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i8, ptr %p0, i64 %idx
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1.ext
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld1 = getelementptr i8, ptr %p1, i64 %idx
+ %ld1 = load i32, ptr %gep.ld1, align 8
+ %ld1.ext = sext i32 %ld1 to i64
+
+ %gep.ld0 = getelementptr i8, ptr %p0, i64 %idx
+ %ld0 = load i64, ptr %gep.ld0, align 8
+
+ %val = add i64 %ld0, %ld1.ext
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+
+; Interleave group with non-constant stride. Probabably doesn't make sense to
+; speculate the stride here (as a heuristic).
+define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'strided_interleave'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%gep.ld0>, ir<1>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i64, ptr %gep.ld0, i61 1
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %ld1 = load i64, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i64, ptr %p, i64 %idx
+ %gep.ld1 = getelementptr i64, ptr %gep.ld0, i61 1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+
+; Mem access ptr in the form of `(%base,+,%stride)<%header>` where %base is
+; something present as in instruction in the loop body (although that would be a
+; SCEV expression, not SCEVUnknown).
+define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset) {
+; CHECK-LABEL: VPlan for loop in 'in_loop_base'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld.base> = getelementptr ir<%p>, ir<%offset>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%gep.ld.base>, ir<%mul>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %mul = mul i64 %iv, %stride
+; CHECK-NEXT: IR %idx = add i64 %mul, %offset
+; CHECK-NEXT: IR %gep.ld.base = getelementptr i64, ptr %p, i64 %offset
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %gep.ld.base, i64 %mul
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %mul = mul i64 %iv, %stride
+ %idx = add i64 %mul, %offset
+
+ %gep.ld.base = getelementptr i64,ptr %p, i64 %offset
+ %gep.ld = getelementptr i64, ptr %gep.ld.base, i64 %mul
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Base (non-recurrent) part of the memory access pointer doesn't have
+; corresponding IR value, e.g., `(base + %iv*%stirde) + %offset`.
+define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset) {
+; CHECK-LABEL: VPlan for loop in 'base_not_in_ir'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = add ir<%mul>, ir<%offset>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %mul = mul i64 %iv, %stride
+; CHECK-NEXT: IR %idx = add i64 %mul, %offset
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %mul = mul i64 %iv, %stride
+ %idx = add i64 %mul, %offset
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; If the base pointer is uniform but not loop-invariant we could still speculate
+; access to be unit-strided, although not implemented yet.
+define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'non_invariant_uniform_base'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%iv.sdiv32> = sdiv ir<%iv>, ir<32>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = add ir<%mul>, ir<%iv.sdiv32>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %iv.sdiv32 = sdiv i64 %iv, 32
+; CHECK-NEXT: IR %mul = mul i64 %iv, %stride
+; CHECK-NEXT: IR %idx = add i64 %mul, %iv.sdiv32
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %iv.sdiv32 = sdiv i64 %iv, 32
+
+ %mul = mul i64 %iv, %stride
+ %idx = add i64 %mul, %iv.sdiv32
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni) {
+; CHECK-LABEL: VPlan for loop in 'non_invariant_uniform_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%iv.sdiv32> = sdiv ir<%iv>, ir<32>
+; CHECK-NEXT: EMIT ir<%gep.uni> = getelementptr ir<%p.uni>, ir<%iv.sdiv32>
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = load ir<%gep.uni>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %iv.sdiv32 = sdiv i64 %iv, 32
+; CHECK-NEXT: IR %gep.uni = getelementptr i64, ptr %p.uni, i64 %iv.sdiv32
+; CHECK-NEXT: IR %stride = load i64, ptr %gep.uni, align 4
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %iv.sdiv32 = sdiv i64 %iv, 32
+
+ %gep.uni = getelementptr i64, ptr %p.uni, i64 %iv.sdiv32
+ %stride = load i64, ptr %gep.uni
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Not valuable by itself, but as a basis for the subsequent test to ensure that
+; non-constant trip count doesn't change anything by itself. Also show the order
+; of checks between stride speculation and trip-count check when not
+; tail-folding.
+define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'non_constant_btc'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %n)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC == stride, so stride speculation would result in zero vector loop
+; iterations.
+define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_as_btc'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %stride)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%stride>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, %stride
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %stride
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above but a slightly more complex dependency between stride and
+; BTC.
+define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_dependent_btc'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %n = add i64 %stride, 1
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (1 + %stride))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %n = add i64 %stride, 1
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC dependent on %stride, but stride speculation doesn't necessarily mean no
+; vector loop iterations. The test shows in which order we emit the run time
+; checks for both.
+define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) {
+; CHECK-LABEL: VPlan for loop in 'stride_btc_checks_order'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %n = mul i64 %m, %stride
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (%stride * %m))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %n = mul i64 %m, %stride
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC fully defined by stride speculation but still allows vector loop execution.
+define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_dependent_btc_non_preventive'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %n = add i64 %stride, 3
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Doesn't pass legality as run-time memory dependencies check doesn't support
+; strided accesses. If it did, the purpose of this test would be to show how all
+; three run-time checks (mem deps/stride speculation/trip-count) would be
+; ordered in respect to each other. Note that pointer aliasing check could
+; potentially be simplified if done after stride speculation. However, that
+; isn't necessarily the best idea because we could also multi-version for stride
+; and keep aliasing part generic and shared by both vector loops..
+define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) {
+entry:
+ %p.out = getelementptr i8, ptr %p, i64 %out.offset
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Same as above but memdep check doesn't depend on stride
+define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
+; CHECK-LABEL: VPlan for loop in 'stride_btc_independent_memdep_triple_check'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %p.out = getelementptr i8, ptr %p2, i64 %out.offset
+; CHECK-NEXT: IR %n = add i64 %stride, 3
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.ld2> = getelementptr ir<%p2>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld2> = load ir<%gep.ld2>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%ld2>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.ld2 = getelementptr i64, ptr %p2, i64 %iv
+; CHECK-NEXT: IR %ld2 = load i64, ptr %gep.ld2, align 8
+; CHECK-NEXT: IR %val = add i64 %ld, %ld2
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %p.out = getelementptr i8, ptr %p2, i64 %out.offset
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.ld2 = getelementptr i64, ptr %p2, i64 %iv
+ %ld2 = load i64, ptr %gep.ld2, align 8
+
+ %val = add i64 %ld, %ld2
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'actual_stride_not_in_ir'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%base> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%base>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %base = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %base, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %base = getelementptr i64, ptr %p, i64 %idx
+ %gep.ld = getelementptr i64, ptr %base, i64 %idx
+
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Gep into multi-dimensional array. Strided last index can be speculated to
+; result in a unit-strided memory access.
+define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'nd_array_last_idx'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<42>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 42, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 42, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+
+; Strided inner index will never result in unit-strided memory-access, even if
+; its stride is one.
+define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'nd_array_non_last_idx'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<42>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 %idx, i64 42
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 %idx, i64 42
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Isn't unit-strided either.
+define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'nd_array_multiple_idxs'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 %idx, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 %idx, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Stride is used through `sext` in the loop.
+define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
+; CHECK-LABEL: VPlan for loop in 'sext_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = sext ir<%stride.i32> to i64
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %stride = sext i32 %stride.i32 to i64
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = sext i32 %stride.i32 to i64
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Stride is used through `trunc` in the loop.
+define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
+; CHECK-LABEL: VPlan for loop in 'trunc_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %stride = trunc i64 %stride.i64 to i32
+; CHECK-NEXT: IR %iv.next = add nsw i32 %iv, 1
+; CHECK-NEXT: IR %idx = mul i32 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i32, ptr %p, i32 %idx
+; CHECK-NEXT: IR %ld = load i32, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+; CHECK-NEXT: IR store i32 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i32 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = trunc i64 %stride.i64 to i32
+ %iv.next = add nsw i32 %iv, 1
+
+ %idx = mul i32 %iv, %stride
+
+ %gep.ld = getelementptr i32, ptr %p, i32 %idx
+ %ld = load i32, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; %stride is usedc through both `trunc`/`sext` for different accesses.
+define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride) {
+; CHECK-LABEL: VPlan for loop in 'trunc_ext_stride'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride.trunc = trunc i32 %stride to i16
+; CHECK-NEXT: IR %stride.ext = sext i32 %stride to i64
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv.trunc> = trunc ir<%iv> to i16
+; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
+; CHECK-NEXT: EMIT ir<%idx.trunc> = mul ir<%iv.trunc>, ir<%stride.trunc>
+; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<%stride.ext>
+; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%p0>, ir<%idx.trunc>
+; CHECK-NEXT: EMIT ir<%gep.ext> = getelementptr ir<%p0>, ir<%idx.ext>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld.trunc> = load ir<%gep.trunc>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld.ext> = load ir<%gep.ext>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld.trunc>, ir<%ld.ext>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i32 %iv, 1
+; CHECK-NEXT: IR %iv.trunc = trunc i32 %iv to i16
+; CHECK-NEXT: IR %iv.ext = sext i32 %iv to i64
+; CHECK-NEXT: IR %idx.trunc = mul i16 %iv.trunc, %stride.trunc
+; CHECK-NEXT: IR %idx.ext = mul i64 %iv.ext, %stride.ext
+; CHECK-NEXT: IR %gep.trunc = getelementptr i32, ptr %p0, i16 %idx.trunc
+; CHECK-NEXT: IR %gep.ext = getelementptr i32, ptr %p0, i64 %idx.ext
+; CHECK-NEXT: IR %ld.trunc = load i32, ptr %gep.trunc, align 4
+; CHECK-NEXT: IR %ld.ext = load i32, ptr %gep.ext, align 4
+; CHECK-NEXT: IR %val = add i32 %ld.trunc, %ld.ext
+; CHECK-NEXT: IR %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+; CHECK-NEXT: IR store i32 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i32 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %stride.trunc = trunc i32 %stride to i16
+ %stride.ext = sext i32 %stride to i64
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i32 %iv, 1
+
+ %iv.trunc = trunc i32 %iv to i16
+ %iv.ext = sext i32 %iv to i64
+
+ %idx.trunc = mul i16 %iv.trunc, %stride.trunc
+ %idx.ext = mul i64 %iv.ext, %stride.ext
+
+ %gep.trunc = getelementptr i32, ptr %p0, i16 %idx.trunc
+ %gep.ext = getelementptr i32, ptr %p0, i64 %idx.ext
+
+ %ld.trunc = load i32, ptr %gep.trunc, align 4
+ %ld.ext = load i32, ptr %gep.ext, align 4
+
+ %val = add i32 %ld.trunc, %ld.ext
+
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Check that we don't speculate unit-strided masked memory access if masked wide
+; memory operation isn't legal (or that we properly pass the mask if it is).
+define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
+; CHECK-LABEL: VPlan for loop in 'basic_masked'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%c> = icmp sge ir<%iv>, ir<%x>
+; CHECK-NEXT: Successor(s): if
+; CHECK-EMPTY:
+; CHECK-NEXT: if:
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>, ir<%c>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>, ir<%c>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>, ir<%c>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %c = icmp sge i64 %iv, %x
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %iv.next = add nsw i64 %iv, 1
+ %c = icmp sge i64 %iv, %x
+ br i1 %c, label %if, label %latch
+
+if:
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+ br label %latch
+
+latch:
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; See https://github.com/llvm/llvm-project/issues/162922.
+define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_poison'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<poison>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, poison
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, poison
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Tests above all used loads, make sure that store is supported too.
+define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'basic_strided_store'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%idx>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %iv
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %idx
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %iv
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %idx
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; This test shows how/if we scalarize address computation def-chain if that
+; pointer has other non-scalar uses.
+define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'ptr_vec_use'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%gep.ptr.st> = getelementptr ir<%p.ptr.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%gep.ld>, ir<%gep.ptr.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
+; CHECK-NEXT: IR %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
+; CHECK-NEXT: IR store ptr %gep.ld, ptr %gep.ptr.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
+ store ptr %gep.ld, ptr %gep.ptr.st
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above, but it's not the resulting pointer itself that has
+; non-scalar use but something in the middle of its def-chain.
+define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_idx_vec_use'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %val = mul i64 %ld, %idx
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %val = mul i64 %ld, %idx
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Another variation of the above, even longer def-chain.
+define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'offset_stride_idx_vec_use'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = add ir<%iv.times.stride>, ir<42>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %iv.times.stride = mul i64 %iv, %stride
+; CHECK-NEXT: IR %idx = add i64 %iv.times.stride, 42
+; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
+; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
+; CHECK-NEXT: IR %val = mul i64 %ld, %idx
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %iv.times.stride = mul i64 %iv, %stride
+ %idx = add i64 %iv.times.stride, 42
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %val = mul i64 %ld, %idx
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; No VPlan dump because `%gep.ld` phi doesn't pass legality currently.
+define void @base_ptr_induction_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64 %stride) {
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %gep.ld = phi ptr [ %p, %entry ], [ %gep.ld.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+ %gep.ld.next = getelementptr inbounds i64, ptr %gep.ld, i64 %stride
+
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
+ store ptr %gep.ld, ptr %gep.ptr.st
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
+; CHECK-LABEL: VPlan for loop in 'test_rewrite_iv_scevs'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %start.ext = zext i32 %start to i64
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV (zext i32 %start to i64)
+; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV (100 + (-1 * (zext i32 %start to i64))<nsw>)<nsw>
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv.0> = WIDEN-INDUCTION ir<%start.ext>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv.1> = WIDEN-INDUCTION ir<0>, vp<[[VP3]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv.1>
+; CHECK-NEXT: EMIT store ir<0.000000e+00>, ir<%gep.dst>
+; CHECK-NEXT: EMIT ir<%iv.1.next> = add ir<%iv.1>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT ir<%iv.0.next> = add ir<%iv.0>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.0.next>, ir<100>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv.0>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%iv.1>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP4]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<%start.ext>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP8]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv.0 = phi i64 [ %start.ext, %entry ], [ %iv.0.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ] (extra operand: vp<%bc.resume.val>.1 from scalar.ph)
+; CHECK-NEXT: IR %gep.dst = getelementptr float, ptr %dst, i64 %iv.1
+; CHECK-NEXT: IR store float 0.000000e+00, ptr %gep.dst, align 4
+; CHECK-NEXT: IR %iv.1.next = add i64 %iv.1, %start.ext
+; CHECK-NEXT: IR %iv.0.next = add i64 %iv.0, 1
+; CHECK-NEXT: IR %ec = icmp eq i64 %iv.0.next, 100
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %start.ext = zext i32 %start to i64
+ br label %loop
+
+loop:
+ %iv.0 = phi i64 [ %start.ext, %entry ], [ %iv.0.next, %loop ]
+ %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ]
+ %gep.dst = getelementptr float, ptr %dst, i64 %iv.1
+ store float 0.0, ptr %gep.dst, align 4
+ %iv.1.next = add i64 %iv.1, %start.ext
+ %iv.0.next = add i64 %iv.0, 1
+ %ec = icmp eq i64 %iv.0.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Keep this in sync with the same under LoopVectorize/
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
new file mode 100644
index 0000000000000..57b2c96507c30
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -0,0 +1,4731 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
+; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV
+
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
+; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
+
+define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @basic(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @basic(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
+; COMPARE-NO-MV-LABEL: define void @basic_optsize(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @basic_optsize(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+attributes #0 = { optsize }
+
+define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
+; COMPARE-NO-MV-LABEL: define void @basic_minsize(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR1:[0-9]+]] {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @basic_minsize(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR1:[0-9]+]] {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+attributes #1 = { minsize }
+
+
+; When using byte-gep with wide memop unit-stride wouldn't be one at the geps
+; index but rather mem-access-type-size. This test has constant multiplier equal
+; to that exact value `8 * %stride` so that `%stride == 1` would result in
+; unit-strided load.
+define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_scaled_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 3)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_scaled_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 3)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 8
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above but constant multiplier is smaller than load type's width, so
+; unit-strideness would require `%stride == 2`.
+define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_under_scaled_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 2)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_under_scaled_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 2)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 4
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Another variation for constant multiplier with byte gep. This time the
+; multiplier is bigger than load access type so this cannot be speculated for
+; unit-strideness.
+define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_over_scaled_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_over_scaled_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 16
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; And another one, with multiplier non-power-of-two. Cannot be speculated for
+; unit-strideness.
+define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_non_power_of_two_scaled_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE]], 11
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_non_power_of_two_scaled_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul i64 [[STRIDE]], 11
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 11
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; No constant multiplier, need to speculate `%stride == sizeof(load-access-type)`.
+define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_nonscaled_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_nonscaled_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; If we negate %stride before indexing, this might be a good heuristic to avoid
+; stride speculation... At the very least, keep this test for the documentation
+; purposes.
+define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_negated_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub i64 0, [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_negated_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 0, [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.neg = sub i64 0, %stride
+ %idx = mul i64 %iv, %stride.neg
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Two memory accesses can be speculated for unit-strideness by single predicate.
+define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @shared_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @shared_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Speculating one access for unit-strideness guarantees that the other one isn't.
+define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @dependent_strides(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP7]]
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP9]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP14]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP15]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> poison, i64 [[TMP27]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 [[TMP30]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @dependent_strides(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP6]]
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[TMP17]]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[STRIDE1:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride1 = add i64 %stride, 1
+ %idx0 = mul i64 %iv, %stride
+ %idx1 = mul i64 %iv, %stride1
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Same as above but in different order. Might be reasonable to have some
+; heuristic to choose one over another instead of just speculating the first
+; access.
+define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @dependent_strides_reverse_order(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP7]]
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP9]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP14]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP15]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> poison, i64 [[TMP27]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 [[TMP30]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @dependent_strides_reverse_order(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP7]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP17]], [[WIDE_LOAD]]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[STRIDE0:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE0]]
+; COMPARE-LAA-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride0 = add i64 %stride, 1
+ %idx0 = mul i64 %iv, %stride0
+ %idx1 = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Two dependent accesses again, but instead of strides being dependent on each
+; other, we have the same stride but different access sizes.
+define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_dependent_byte_geps(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP20]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP21]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP23]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = sext <4 x i32> [[TMP28]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = add <4 x i64> [[TMP16]], [[TMP29]]
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP30]], ptr [[TMP31]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_dependent_byte_geps(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP19]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> poison, i32 [[TMP20]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP23]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = sext <4 x i32> [[TMP27]] to <4 x i64>
+; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP15]], [[TMP28]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i8, ptr %p0, i64 %idx
+ %ld0 = load i64, ptr %gep.ld0, align 8
+
+ %gep.ld1 = getelementptr i8, ptr %p1, i64 %idx
+ %ld1 = load i32, ptr %gep.ld1, align 8
+ %ld1.ext = sext i32 %ld1 to i64
+
+ %val = add i64 %ld0, %ld1.ext
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Likewise but reverse order of accesses.
+define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_dependent_byte_geps_reverse_order(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = sext <4 x i32> [[TMP16]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP20]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = load i64, ptr [[TMP21]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> poison, i64 [[TMP22]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP25]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = add <4 x i64> [[TMP29]], [[TMP17]]
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP30]], ptr [[TMP31]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_dependent_byte_geps_reverse_order(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = sext <4 x i32> [[TMP15]] to <4 x i64>
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP19]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP20]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP28]], [[TMP16]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld1 = getelementptr i8, ptr %p1, i64 %idx
+ %ld1 = load i32, ptr %gep.ld1, align 8
+ %ld1.ext = sext i32 %ld1 to i64
+
+ %gep.ld0 = getelementptr i8, ptr %p0, i64 %idx
+ %ld0 = load i64, ptr %gep.ld0, align 8
+
+ %val = add i64 %ld0, %ld1.ext
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+
+; Interleave group with non-constant stride. Probabably doesn't make sense to
+; speculate the stride here (as a heuristic).
+define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @strided_interleave(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP5]], i61 1
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP6]], i61 1
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP7]], i61 1
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP8]], i61 1
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @strided_interleave(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i61 1
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[GEP_LD0]], i61 1
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld0 = getelementptr i64, ptr %p, i64 %idx
+ %gep.ld1 = getelementptr i64, ptr %gep.ld0, i61 1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+
+; Mem access ptr in the form of `(%base,+,%stride)<%header>` where %base is
+; something present as in instruction in the loop body (although that would be a
+; SCEV expression, not SCEVUnknown).
+define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset) {
+; COMPARE-NO-MV-LABEL: define void @in_loop_base(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @in_loop_base(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = add i64 [[MUL]], [[OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD_BASE:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[GEP_LD_BASE]], i64 [[MUL]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %mul = mul i64 %iv, %stride
+ %idx = add i64 %mul, %offset
+
+ %gep.ld.base = getelementptr i64,ptr %p, i64 %offset
+ %gep.ld = getelementptr i64, ptr %gep.ld.base, i64 %mul
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Base (non-recurrent) part of the memory access pointer doesn't have
+; corresponding IR value, e.g., `(base + %iv*%stirde) + %offset`.
+define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset) {
+; COMPARE-NO-MV-LABEL: define void @base_not_in_ir(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[OFFSET]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add <4 x i64> [[TMP0]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @base_not_in_ir(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = add i64 [[MUL]], [[OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %mul = mul i64 %iv, %stride
+ %idx = add i64 %mul, %offset
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; If the base pointer is uniform but not loop-invariant we could still speculate
+; access to be unit-strided, although not implemented yet.
+define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @non_invariant_uniform_base(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sdiv <4 x i64> [[VEC_IND]], splat (i64 32)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[TMP1]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP6]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @non_invariant_uniform_base(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sdiv <4 x i64> [[VEC_IND]], splat (i64 32)
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[TMP1]], [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP6]]
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %iv.sdiv32 = sdiv i64 %iv, 32
+
+ %mul = mul i64 %iv, %stride
+ %idx = add i64 %mul, %iv.sdiv32
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni) {
+; COMPARE-NO-MV-LABEL: define void @non_invariant_uniform_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], ptr [[P_UNI:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sdiv i64 [[INDEX]], 32
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_UNI]], i64 [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP6]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP19]], ptr [[TMP20]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @non_invariant_uniform_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], ptr [[P_UNI:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sdiv i64 [[INDEX]], 32
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_UNI]], i64 [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP6]]
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP19]], ptr [[TMP20]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %iv.sdiv32 = sdiv i64 %iv, 32
+
+ %gep.uni = getelementptr i64, ptr %p.uni, i64 %iv.sdiv32
+ %stride = load i64, ptr %gep.uni
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Not valuable by itself, but as a basis for the subsequent test to ensure that
+; non-constant trip count doesn't change anything by itself. Also show the order
+; of checks between stride speculation and trip-count check when not
+; tail-folding.
+define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
+; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC == stride, so stride speculation would result in zero vector loop
+; iterations.
+define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_as_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %stride
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above but a slightly more complex dependency between stride and
+; BTC.
+define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %n = add i64 %stride, 1
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC dependent on %stride, but stride speculation doesn't necessarily mean no
+; vector loop iterations. The test shows in which order we emit the run time
+; checks for both.
+define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_checks_order(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_btc_checks_order(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX1]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX1]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %n = mul i64 %m, %stride
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; BTC fully defined by stride speculation but still allows vector loop execution.
+define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc_non_preventive(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Doesn't pass legality as run-time memory dependencies check doesn't support
+; strided accesses. If it did, the purpose of this test would be to show how all
+; three run-time checks (mem deps/stride speculation/trip-count) would be
+; ordered in respect to each other. Note that pointer aliasing check could
+; potentially be simplified if done after stride speculation. However, that
+; isn't necessarily the best idea because we could also multi-version for stride
+; and keep aliasing part generic and shared by both vector loops..
+define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
+; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
+; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %p.out = getelementptr i8, ptr %p, i64 %out.offset
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Same as above but memdep check doesn't depend on stride
+define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
+; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
+; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[P2]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP0]], ptr [[P_OUT]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %p.out = getelementptr i8, ptr %p2, i64 %out.offset
+ %n = add i64 %stride, 3
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.ld2 = getelementptr i64, ptr %p2, i64 %iv
+ %ld2 = load i64, ptr %gep.ld2, align 8
+
+ %val = add i64 %ld, %ld2
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, %n
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @actual_stride_not_in_ir(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP5]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP6]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP8]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP20]], ptr [[TMP21]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @actual_stride_not_in_ir(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP5]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP6]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP8]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP20]], ptr [[TMP21]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %base = getelementptr i64, ptr %p, i64 %idx
+ %gep.ld = getelementptr i64, ptr %base, i64 %idx
+
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Gep into multi-dimensional array. Strided last index can be speculated to
+; result in a unit-strided memory access.
+define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @nd_array_last_idx(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @nd_array_last_idx(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 42, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+
+; Strided inner index will never result in unit-strided memory-access, even if
+; its stride is one.
+define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @nd_array_non_last_idx(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP1]], i64 42
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP2]], i64 42
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP3]], i64 42
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP4]], i64 42
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @nd_array_non_last_idx(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[INDEX]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP1]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP2]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP3]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[IDX]], i64 42
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 %idx, i64 42
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Isn't unit-strided either.
+define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @nd_array_multiple_idxs(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP1]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP2]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP3]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP4]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @nd_array_multiple_idxs(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP1]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP2]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP3]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP4]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr [256 x [256 x i64]], ptr %p, i64 1, i64 %idx, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Stride is used through `sext` in the loop.
+define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
+; COMPARE-NO-MV-LABEL: define void @sext_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[STRIDE_I32:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STRIDE_I32]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sext <4 x i32> [[BROADCAST_SPLAT]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @sext_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[STRIDE_I32:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = sext i32 [[STRIDE_I32]] to i64
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = sext i32 %stride.i32 to i64
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Stride is used through `trunc` in the loop.
+define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
+; COMPARE-NO-MV-LABEL: define void @trunc_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = trunc i64 [[STRIDE_I64]] to i32
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = sub i32 0, [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 0
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP3]], i32 127)
+; COMPARE-NO-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; COMPARE-NO-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = sub i32 0, [[MUL_RESULT]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp slt i32 [[MUL_RESULT]], 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_I64]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i32>
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = mul <4 x i32> [[VEC_IND]], [[TMP9]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP10]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP11]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP10]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP12]]
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP10]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP13]]
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP10]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP14]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP26]], ptr [[TMP27]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i32, ptr [[P]], i32 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i32, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
+; COMPARE-NO-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @trunc_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i32, ptr [[P]], i32 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i32, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = trunc i64 %stride.i64 to i32
+ %iv.next = add nsw i32 %iv, 1
+
+ %idx = mul i32 %iv, %stride
+
+ %gep.ld = getelementptr i32, ptr %p, i32 %idx
+ %ld = load i32, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; %stride is usedc through both `trunc`/`sext` for different accesses.
+define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride) {
+; COMPARE-NO-MV-LABEL: define void @trunc_ext_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i32 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i32 [[STRIDE]] to i16
+; COMPARE-NO-MV-NEXT: [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub i16 0, [[STRIDE_TRUNC]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = icmp slt i16 [[STRIDE_TRUNC]], 0
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i16 [[TMP0]], i16 [[STRIDE_TRUNC]]
+; COMPARE-NO-MV-NEXT: [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[TMP2]], i16 127)
+; COMPARE-NO-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
+; COMPARE-NO-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = sub i16 0, [[MUL_RESULT]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = icmp slt i16 [[MUL_RESULT]], 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp sgt i16 [[TMP3]], 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = select i1 [[TMP1]], i1 [[TMP5]], i1 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[MUL_OVERFLOW]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[STRIDE_TRUNC]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_EXT]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND3:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = sext <4 x i32> [[VEC_IND]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul <4 x i16> [[VEC_IND3]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = mul <4 x i64> [[TMP8]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[TMP9]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP10]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i16> [[TMP9]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP11]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i16> [[TMP9]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP12]]
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = extractelement <4 x i16> [[TMP9]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP13]]
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP14]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP15]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP14]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP16]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP14]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP17]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP14]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP18]]
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP19]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP20]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP21]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP22]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP23]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP24]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP25]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP26]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> poison, i32 [[TMP35]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP37]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP38]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP43:%.*]] = add <4 x i32> [[TMP34]], [[TMP42]]
+; COMPARE-NO-MV-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP43]], ptr [[TMP44]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT4]] = add <4 x i16> [[VEC_IND3]], splat (i16 4)
+; COMPARE-NO-MV-NEXT: [[TMP45:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IV_TRUNC:%.*]] = trunc i32 [[IV]] to i16
+; COMPARE-NO-MV-NEXT: [[IV_EXT:%.*]] = sext i32 [[IV]] to i64
+; COMPARE-NO-MV-NEXT: [[IDX_TRUNC:%.*]] = mul i16 [[IV_TRUNC]], [[STRIDE_TRUNC]]
+; COMPARE-NO-MV-NEXT: [[IDX_EXT:%.*]] = mul i64 [[IV_EXT]], [[STRIDE_EXT]]
+; COMPARE-NO-MV-NEXT: [[GEP_TRUNC:%.*]] = getelementptr i32, ptr [[P0]], i16 [[IDX_TRUNC]]
+; COMPARE-NO-MV-NEXT: [[GEP_EXT:%.*]] = getelementptr i32, ptr [[P0]], i64 [[IDX_EXT]]
+; COMPARE-NO-MV-NEXT: [[LD_TRUNC:%.*]] = load i32, ptr [[GEP_TRUNC]], align 4
+; COMPARE-NO-MV-NEXT: [[LD_EXT:%.*]] = load i32, ptr [[GEP_EXT]], align 4
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i32 [[LD_TRUNC]], [[LD_EXT]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
+; COMPARE-NO-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @trunc_ext_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i32 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i32 [[STRIDE]] to i16
+; COMPARE-LAA-MV-NEXT: [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = mul i16 [[TMP0]], [[STRIDE_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IV_TRUNC:%.*]] = trunc i32 [[IV]] to i16
+; COMPARE-LAA-MV-NEXT: [[IV_EXT:%.*]] = sext i32 [[IV]] to i64
+; COMPARE-LAA-MV-NEXT: [[IDX_TRUNC:%.*]] = mul i16 [[IV_TRUNC]], [[STRIDE_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[IDX_EXT:%.*]] = mul i64 [[IV_EXT]], [[STRIDE_EXT]]
+; COMPARE-LAA-MV-NEXT: [[GEP_TRUNC:%.*]] = getelementptr i32, ptr [[P0]], i16 [[IDX_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[GEP_EXT:%.*]] = getelementptr i32, ptr [[P0]], i64 [[IDX_EXT]]
+; COMPARE-LAA-MV-NEXT: [[LD_TRUNC:%.*]] = load i32, ptr [[GEP_TRUNC]], align 4
+; COMPARE-LAA-MV-NEXT: [[LD_EXT:%.*]] = load i32, ptr [[GEP_EXT]], align 4
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i32 [[LD_TRUNC]], [[LD_EXT]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %stride.trunc = trunc i32 %stride to i16
+ %stride.ext = sext i32 %stride to i64
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i32 %iv, 1
+
+ %iv.trunc = trunc i32 %iv to i16
+ %iv.ext = sext i32 %iv to i64
+
+ %idx.trunc = mul i16 %iv.trunc, %stride.trunc
+ %idx.ext = mul i64 %iv.ext, %stride.ext
+
+ %gep.trunc = getelementptr i32, ptr %p0, i16 %idx.trunc
+ %gep.ext = getelementptr i32, ptr %p0, i64 %idx.ext
+
+ %ld.trunc = load i32, ptr %gep.trunc, align 4
+ %ld.ext = load i32, ptr %gep.ext, align 4
+
+ %val = add i32 %ld.trunc, %ld.ext
+
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Check that we don't speculate unit-strided masked memory access if masked wide
+; memory operation isn't legal (or that we properly pass the mask if it is).
+define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
+; COMPARE-NO-MV-LABEL: define void @basic_masked(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[X:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = icmp sge <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-NO-MV: [[PRED_STORE_IF]]:
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP6]]
+; COMPARE-NO-MV-NEXT: store i64 [[TMP5]], ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-NO-MV: [[PRED_STORE_CONTINUE]]:
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-NO-MV: [[PRED_STORE_IF3]]:
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = add i64 [[TMP6]], 1
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-NO-MV-NEXT: store i64 [[TMP11]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-NO-MV: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMPARE-NO-MV: [[PRED_STORE_IF5]]:
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP15]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add i64 [[TMP6]], 2
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP18]]
+; COMPARE-NO-MV-NEXT: store i64 [[TMP17]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-NO-MV: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV: [[PRED_STORE_IF7]]:
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP21]]
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = add i64 [[TMP6]], 3
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
+; COMPARE-NO-MV-NEXT: store i64 [[TMP23]], ptr [[TMP25]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMPARE-NO-MV: [[PRED_STORE_CONTINUE8]]:
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP6]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @basic_masked(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[X:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = icmp sge <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = add i64 [[TMP2]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP7]]
+; COMPARE-LAA-MV-NEXT: store i64 [[TMP9]], ptr [[TMP10]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = add i64 [[TMP2]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-LAA-MV-NEXT: store i64 [[TMP14]], ptr [[TMP15]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = add i64 [[TMP2]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
+; COMPARE-LAA-MV-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[C:%.*]] = icmp sge i64 [[IV]], [[X]]
+; COMPARE-LAA-MV-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[LATCH]]
+; COMPARE-LAA-MV: [[IF]]:
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[LATCH]]
+; COMPARE-LAA-MV: [[LATCH]]:
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP51:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %iv.next = add nsw i64 %iv, 1
+ %c = icmp sge i64 %iv, %x
+ br i1 %c, label %if, label %latch
+
+if:
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+ br label %latch
+
+latch:
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; See https://github.com/llvm/llvm-project/issues/162922.
+define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_poison(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], poison
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_poison(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], poison
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, poison
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Tests above all used loads, make sure that store is supported too.
+define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @basic_strided_store(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 0
+; COMPARE-NO-MV-NEXT: store i64 [[TMP6]], ptr [[TMP10]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 1
+; COMPARE-NO-MV-NEXT: store i64 [[TMP7]], ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 2
+; COMPARE-NO-MV-NEXT: store i64 [[TMP8]], ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 3
+; COMPARE-NO-MV-NEXT: store i64 [[TMP9]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @basic_strided_store(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %iv
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %idx
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; This test shows how/if we scalarize address computation def-chain if that
+; pointer has other non-scalar uses.
+define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @ptr_vec_use(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP2]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP3]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP7]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> [[TMP11]], i64 [[TMP8]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP14]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x ptr> [[TMP1]], ptr [[TMP15]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @ptr_vec_use(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x ptr> [[TMP0]], ptr [[TMP3]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
+ store ptr %gep.ld, ptr %gep.ptr.st
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above, but it's not the resulting pointer itself that has
+; non-scalar use but something in the middle of its def-chain.
+define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_idx_vec_use(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = mul <4 x i64> [[TMP16]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_idx_vec_use(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = mul i64 [[LD]], [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %val = mul i64 %ld, %idx
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Another variation of the above, even longer def-chain.
+define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @offset_stride_idx_vec_use(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add <4 x i64> [[TMP0]], splat (i64 42)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[TMP17]], [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @offset_stride_idx_vec_use(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 42)
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IV_TIMES_STRIDE:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = add i64 [[IV_TIMES_STRIDE]], 42
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = mul i64 [[LD]], [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %iv.times.stride = mul i64 %iv, %stride
+ %idx = add i64 %iv.times.stride, 42
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %val = mul i64 %ld, %idx
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; No VPlan dump because `%gep.ld` phi doesn't pass legality currently.
+define void @base_ptr_induction_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @base_ptr_induction_vec_use(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = phi ptr [ [[P]], %[[ENTRY]] ], [ [[GEP_LD_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[GEP_LD_NEXT]] = getelementptr inbounds i64, ptr [[GEP_LD]], i64 [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @base_ptr_induction_vec_use(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = phi ptr [ [[P]], %[[ENTRY]] ], [ [[GEP_LD_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[GEP_LD_NEXT]] = getelementptr inbounds i64, ptr [[GEP_LD]], i64 [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %gep.ld = phi ptr [ %p, %entry ], [ %gep.ld.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+ %gep.ld.next = getelementptr inbounds i64, ptr %gep.ld, i64 %stride
+
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
+ store ptr %gep.ld, ptr %gep.ptr.st
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
+; COMPARE-NO-MV-LABEL: define void @test_rewrite_iv_scevs(
+; COMPARE-NO-MV-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub nsw i64 100, [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[START_EXT]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = mul i64 1, [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = mul i64 2, [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], [[TMP7]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul i64 3, [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], [[TMP9]]
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[DST]], i64 [[OFFSET_IDX]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP6]]
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP10]]
+; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP11]], align 4
+; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP12]], align 4
+; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP13]], align 4
+; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP14]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: br label %[[LOOP:.*]]
+; COMPARE-NO-MV: [[LOOP]]:
+; COMPARE-NO-MV-NEXT: [[IV_0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_0_NEXT:%.*]], %[[LOOP]] ]
+; COMPARE-NO-MV-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
+; COMPARE-NO-MV-NEXT: [[GEP_DST:%.*]] = getelementptr float, ptr [[DST]], i64 [[IV_1]]
+; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[GEP_DST]], align 4
+; COMPARE-NO-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
+; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @test_rewrite_iv_scevs(
+; COMPARE-LAA-MV-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 100, [[START_EXT]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[START]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[DST]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 97, %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[LOOP:.*]]
+; COMPARE-LAA-MV: [[LOOP]]:
+; COMPARE-LAA-MV-NEXT: [[IV_0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_0_NEXT:%.*]], %[[LOOP]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
+; COMPARE-LAA-MV-NEXT: [[GEP_DST:%.*]] = getelementptr float, ptr [[DST]], i64 [[IV_1]]
+; COMPARE-LAA-MV-NEXT: store float 0.000000e+00, ptr [[GEP_DST]], align 4
+; COMPARE-LAA-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
+; COMPARE-LAA-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
+; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP62:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %start.ext = zext i32 %start to i64
+ br label %loop
+
+loop:
+ %iv.0 = phi i64 [ %start.ext, %entry ], [ %iv.0.next, %loop ]
+ %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ]
+ %gep.dst = getelementptr float, ptr %dst, i64 %iv.1
+ store float 0.0, ptr %gep.dst, align 4
+ %iv.1.next = add i64 %iv.1, %start.ext
+ %iv.0.next = add i64 %iv.0, 1
+ %ec = icmp eq i64 %iv.0.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Keep this in sync with the same under VPlan/
>From 662b2c6a8751d5ca8549d06c330bfd3c4320b4ac Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 16 Jul 2026 16:56:17 -0700
Subject: [PATCH 02/13] Add `@independent_strides` test
---
.../VPlan/vplan-based-stride-mv.ll | 92 ++++++
.../LoopVectorize/vplan-based-stride-mv.ll | 303 +++++++++++++-----
2 files changed, 309 insertions(+), 86 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index 09f4b71594185..db4d2ce2738a2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -837,6 +837,98 @@ exit:
ret void
}
+; Both strides should be speculated and combined into a single predicate.
+define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride0, i64 %stride1) {
+; CHECK-LABEL: VPlan for loop in 'independent_strides'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
+; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
+; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
+; CHECK-NEXT: IR %idx0 = mul i64 %iv, %stride0
+; CHECK-NEXT: IR %idx1 = mul i64 %iv, %stride1
+; CHECK-NEXT: IR %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+; CHECK-NEXT: IR %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+; CHECK-NEXT: IR %ld0 = load i64, ptr %gep.ld0, align 8
+; CHECK-NEXT: IR %ld1 = load i64, ptr %gep.ld1, align 8
+; CHECK-NEXT: IR %val = add i64 %ld0, %ld1
+; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+; CHECK-NEXT: IR store i64 %val, ptr %gep.st, align 8
+; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx0 = mul i64 %iv, %stride0
+ %idx1 = mul i64 %iv, %stride1
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
; Speculating one access for unit-strideness guarantees that the other one isn't.
define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-LABEL: VPlan for loop in 'dependent_strides'
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 57b2c96507c30..4429b0e731616 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -1075,6 +1075,137 @@ exit:
ret void
}
+; Both strides should be speculated and combined into a single predicate.
+define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride0, i64 %stride1) {
+; COMPARE-NO-MV-LABEL: define void @independent_strides(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE0]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE1]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP6]]
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP12]]
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP14]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP16]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP3]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP19]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP11]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP15]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> poison, i64 [[TMP26]], i32 0
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP30]], i64 [[TMP27]], i32 1
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 2
+; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = add <4 x i64> [[TMP25]], [[TMP33]]
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP34]], ptr [[TMP35]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @independent_strides(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE0]], 1
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE1]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP0]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE0]]
+; COMPARE-LAA-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx0 = mul i64 %iv, %stride0
+ %idx1 = mul i64 %iv, %stride1
+
+ %gep.ld0 = getelementptr i64, ptr %p0, i64 %idx0
+ %gep.ld1 = getelementptr i64, ptr %p1, i64 %idx1
+ %ld0 = load i64, ptr %gep.ld0, align 8
+ %ld1 = load i64, ptr %gep.ld1, align 8
+ %val = add i64 %ld0, %ld1
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
; Speculating one access for unit-strideness guarantees that the other one isn't.
define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @dependent_strides(
@@ -1129,7 +1260,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1172,7 +1303,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1191,7 +1322,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1278,7 +1409,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1321,7 +1452,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1340,7 +1471,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1421,7 +1552,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1471,7 +1602,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1489,7 +1620,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1570,7 +1701,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1620,7 +1751,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1638,7 +1769,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1720,7 +1851,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1746,7 +1877,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1763,7 +1894,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1831,7 +1962,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1855,7 +1986,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1871,7 +2002,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1938,7 +2069,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1962,7 +2093,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1977,7 +2108,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2042,7 +2173,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2083,7 +2214,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2151,7 +2282,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2193,7 +2324,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2268,7 +2399,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2284,7 +2415,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2309,7 +2440,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2325,7 +2456,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2391,7 +2522,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2407,7 +2538,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2491,7 +2622,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2507,7 +2638,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2594,7 +2725,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2610,7 +2741,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2637,7 +2768,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2653,7 +2784,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2720,7 +2851,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2736,7 +2867,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2768,7 +2899,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2856,7 +2987,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2932,7 +3063,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2951,7 +3082,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2994,7 +3125,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3065,7 +3196,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3108,7 +3239,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -3174,7 +3305,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3197,7 +3328,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3211,7 +3342,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3274,7 +3405,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3310,7 +3441,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3324,7 +3455,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3385,7 +3516,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3424,7 +3555,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -3488,7 +3619,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3511,7 +3642,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3526,7 +3657,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP47:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3603,7 +3734,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
@@ -3618,7 +3749,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-NO-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3639,7 +3770,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3654,7 +3785,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP49:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3754,7 +3885,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT4]] = add <4 x i16> [[VEC_IND3]], splat (i16 4)
; COMPARE-NO-MV-NEXT: [[TMP45:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
@@ -3774,7 +3905,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-NO-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3803,7 +3934,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3823,7 +3954,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP51:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3924,7 +4055,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP6]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3987,7 +4118,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4006,7 +4137,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: br label %[[LATCH]]
; COMPARE-LAA-MV: [[LATCH]]:
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP51:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4070,7 +4201,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4107,7 +4238,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -4170,7 +4301,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4193,7 +4324,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4207,7 +4338,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4268,7 +4399,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4296,7 +4427,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4312,7 +4443,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4378,7 +4509,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4404,7 +4535,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4419,7 +4550,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4484,7 +4615,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4512,7 +4643,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4528,7 +4659,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4654,7 +4785,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP14]], align 4
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -4670,7 +4801,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP50:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -4691,7 +4822,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 4
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4706,7 +4837,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-LAA-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP62:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP64:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
>From 193fc77621feef3dfa7bede8ddb650e51661050c Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 25 Feb 2026 10:01:33 -0800
Subject: [PATCH 03/13] [VPlan] Implement VPlan-based stride speculation
---
.../Vectorize/LoopVectorizationPlanner.h | 6 +
.../Transforms/Vectorize/LoopVectorize.cpp | 10 +
.../Transforms/Vectorize/VPRecipeBuilder.h | 12 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 251 ++-
.../Transforms/Vectorize/VPlanTransforms.h | 6 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 5 +-
.../VPlan/vplan-based-stride-mv.ll | 621 +++---
.../LoopVectorize/vplan-based-stride-mv.ll | 1672 ++++++++---------
11 files changed, 1478 insertions(+), 1158 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index cd02ced2942d1..f6c5da8126c7f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -516,6 +516,12 @@ class VPBuilder {
return tryInsertInstruction(new VPExpandSCEVRecipe(Expr));
}
+ VPExpandStridePredicatesRecipe *
+ createExpandSCEVPredicate(const SCEVUnionPredicate &StridePredicates) {
+ return tryInsertInstruction(new VPExpandStridePredicatesRecipe(
+ StridePredicates, this->getPlan().getContext()));
+ }
+
VPVectorPointerRecipe *
createVectorPointer(VPValue *Ptr, Type *SourceElementTy, VPValue *Stride,
GEPNoWrapFlags GEPFlags, DebugLoc DL) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1929f52ae7d95..118d4402cbf37 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6375,6 +6375,16 @@ bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
return false;
}
+bool VPRecipeBuilder::isConsecutiveWithoutVPlanBasedStrideSpeculation(
+ VPInstruction *MemOp) {
+ auto *I = MemOp->getUnderlyingInstr();
+ auto *PtrOp = getLoadStorePointerOperand(I);
+ auto *ScalarTy = MemOp->getOpcode() == Instruction::Load
+ ? I->getType()
+ : I->getOperand(0)->getType();
+ return Legal->isConsecutivePtr(ScalarTy, PtrOp);
+}
+
VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
VFRange &Range) {
auto *I = VPI->getUnderlyingInstr();
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 1303b62d5faf4..0281192d68f3b 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -33,11 +33,6 @@ class VPRecipeBuilder {
VPBuilder &Builder;
- /// Check if \p I can be widened at the start of \p Range and possibly
- /// decrease the range such that the returned value holds for the entire \p
- /// Range. The function should not be called for memory instructions or calls.
- bool shouldWiden(Instruction *I, VFRange &Range) const;
-
/// Optimize the special case where the operand of \p VPI is a constant
/// integer induction variable.
VPWidenIntOrFpInductionRecipe *
@@ -61,6 +56,11 @@ class VPRecipeBuilder {
/// Returns true if the target prefers vectorized addressing.
bool prefersVectorizedAddressing() const;
+ /// Check if \p I can be widened at the start of \p Range and possibly
+ /// decrease the range such that the returned value holds for the entire \p
+ /// Range. The function should not be called for memory instructions or calls.
+ bool shouldWiden(Instruction *I, VFRange &Range) const;
+
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
@@ -90,6 +90,8 @@ class VPRecipeBuilder {
/// predicated, add the mask as last operand. Range.End may be decreased to
/// ensure same recipe behavior from \p Range.Start to \p Range.End.
VPSingleDefRecipe *handleReplication(VPInstruction *VPI, VFRange &Range);
+
+ bool isConsecutiveWithoutVPlanBasedStrideSpeculation(VPInstruction *MemOp);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 7d69b3453cba9..10f957609f238 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -677,7 +677,9 @@ void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
auto RecipeIndent = Indent + " ";
for (const VPRecipeBase &Recipe : *this) {
Recipe.print(O, RecipeIndent, SlotTracker);
- O << '\n';
+ // SCEVPredicate::print adds a newline so we don't want to add one for it:
+ if (!isa<VPExpandStridePredicatesRecipe>(Recipe))
+ O << '\n';
}
printSuccessors(O, Indent);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b67d1a1281c14..64ab410a77b53 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -418,6 +418,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPBranchOnMaskSC,
VPDerivedIVSC,
VPExpandSCEVSC,
+ VPExpandStridePredicatesSC,
VPExpressionSC,
VPIRInstructionSC,
VPInstructionSC,
@@ -625,6 +626,7 @@ class LLVM_ABI_FOR_TEST VPSingleDefRecipe : public VPRecipeBase,
switch (R->getVPRecipeID()) {
case VPRecipeBase::VPDerivedIVSC:
case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPExpandStridePredicatesSC:
case VPRecipeBase::VPExpressionSC:
case VPRecipeBase::VPInstructionSC:
case VPRecipeBase::VPReductionEVLSC:
@@ -4021,6 +4023,46 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
#endif
};
+class VPExpandStridePredicatesRecipe : public VPSingleDefRecipe {
+ SCEVUnionPredicate StridePredicates;
+
+public:
+ VPExpandStridePredicatesRecipe(const SCEVUnionPredicate &StridePredicates,
+ LLVMContext &Ctx)
+ : VPSingleDefRecipe(VPRecipeBase::VPExpandStridePredicatesSC, {},
+ Type::getInt1Ty(Ctx)),
+ StridePredicates(StridePredicates) {}
+
+ ~VPExpandStridePredicatesRecipe() override = default;
+
+ VPExpandStridePredicatesRecipe *clone() override {
+ return new VPExpandStridePredicatesRecipe(StridePredicates,
+ getScalarType()->getContext());
+ }
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPExpandStridePredicatesSC)
+
+ void execute(VPTransformState &State) override {
+ llvm_unreachable("SCEVPredicates must be expanded before final execute");
+ }
+
+ /// Return the cost of this VPExpandSCEVRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override {
+ // TODO: Compute accurate cost after retiring the legacy cost model.
+ return 0;
+ }
+
+ const SCEVPredicate *getSCEVPredicate() const { return &StridePredicates; }
+
+protected:
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for generating the active lane mask for the vector loop that is
/// used to predicate the vector operations.
class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c62c4a082b95e..75fb9b3734e94 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -4824,6 +4824,13 @@ void VPExpandSCEVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
printAsOperand(O, SlotTracker);
O << " = EXPAND SCEV " << *Expr;
}
+
+void VPExpandStridePredicatesRecipe::printRecipe(
+ raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const {
+ O << Indent << "EMIT ";
+ printAsOperand(O, SlotTracker);
+ O << " = EXPAND SCEVPredicate " << StridePredicates;
+}
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1dc13889af4d6..6d556a3bd29be 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -48,6 +48,11 @@ using namespace llvm;
using namespace VPlanPatternMatch;
using namespace SCEVPatternMatch;
+static cl::opt<bool> EnableVPlanBasedStrideMV(
+ "enable-vplan-based-stride-mv", cl::init(false), cl::Hidden,
+ cl::desc("Perform stride multiversioning directly on VPlan instead of in "
+ "LoopAccessAnalysis."));
+
bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
VPlan &Plan, const TargetLibraryInfo &TLI) {
@@ -5830,6 +5835,17 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
DenseMap<const SCEV *, Value *> ExpandedSCEVs;
// Expand remaining VPExpandSCEVRecipes to IR instructions using SCEVExpander.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
+ if (auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(&R)) {
+ Value *Res = Expander.expandCodeForPredicate(
+ ExpStrides->getSCEVPredicate(), EntryBB->getTerminator());
+ Res->setName("strides.mv.check");
+ VPValue *Exp = Plan.getOrAddLiveIn(Res);
+
+ ExpStrides->replaceAllUsesWith(Exp);
+ ExpStrides->eraseFromParent();
+ continue;
+ }
+
auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
if (!ExpSCEV)
continue;
@@ -5843,8 +5859,10 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
Plan.resetTripCount(Exp);
ExpSCEV->eraseFromParent();
}
- assert(none_of(*Entry, IsaPred<VPExpandSCEVRecipe>) &&
- "all VPExpandSCEVRecipes must have been expanded");
+ assert(none_of(*Entry,
+ IsaPred<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>) &&
+ "all VPExpandSCEVRecipes/VPExpandStridePredicatesRecipe must have "
+ "been expanded");
// Add IR instructions in the entry basic block but not in the VPIRBasicBlock
// to the VPIRBasicBlock.
auto EI = Entry->begin();
@@ -7474,6 +7492,10 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
return ReplaceWith(VPI, WidenedR);
});
+ if (EnableVPlanBasedStrideMV)
+ RUN_VPLAN_PASS(VPlanTransforms::multiversionForUnitStridedMemOps, Plan,
+ CostCtx, RecipeBuilder, Range, MemOps);
+
VPlanTransforms::runPass("delegateMemOpWideningToLegacyCM", ProcessSubset,
Plan, [&](VPInstruction *VPI) {
if (VPRecipeBase *Recipe =
@@ -7484,6 +7506,231 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
});
}
+void VPlanTransforms::multiversionForUnitStridedMemOps(
+ VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps) {
+ if (CostCtx.L->getHeader()->getParent()->hasOptSize())
+ return;
+ SmallVector<VPInstruction *> RemainingOps;
+
+ ScalarEvolution *SE = CostCtx.PSE.getSE();
+
+ PredicatedScalarEvolution StrideMVPSE(*SE, const_cast<Loop &>(*CostCtx.L));
+
+ SCEVUnionPredicate StridePredicates({}, *SE);
+
+ // Use `for_each` so that we could do `return Skip();`.
+ for_each(MemOps, [&](VPInstruction *VPI) {
+ auto Skip = [&]() { RemainingOps.push_back(VPI); };
+ if (RecipeBuilder.isConsecutiveWithoutVPlanBasedStrideSpeculation(VPI))
+ return Skip();
+ auto *PtrOp = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
+ : VPI->getOperand(1);
+
+ const SCEV *PtrSCEV =
+ vputils::getSCEVExprForVPValue(PtrOp, CostCtx.PSE, CostCtx.L);
+ const SCEV *Start = nullptr;
+ const SCEV *Stride = nullptr;
+
+ if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
+ m_SpecificLoop(CostCtx.L)))) {
+ return Skip();
+ }
+
+ Type *ScalarTy = VPI->getOpcode() == Instruction::Load
+ ? VPI->getScalarType()
+ : VPI->getOperand(0)->getScalarType();
+
+ if (VPI->getMask()) {
+ auto &TTI = CostCtx.TTI;
+ Instruction *I = VPI->getUnderlyingInstr();
+ unsigned AS = getLoadStoreAddressSpace(I);
+ const Align Alignment = getLoadStoreAlignment(I);
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) -> bool {
+ Type *VTy = VectorType::get(ScalarTy, VF);
+ return VPI->getOpcode() == Instruction::Load
+ ? (TTI.isLegalMaskedLoad(VTy, Alignment, AS) ||
+ TTI.isLegalMaskedGather(VTy, Alignment))
+ : (TTI.isLegalMaskedStore(VTy, Alignment, AS) ||
+ TTI.isLegalMaskedScatter(VTy, Alignment));
+ },
+ Range))
+ return Skip();
+ }
+
+ const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
+ Stride->getType(), SE->getDataLayout().getTypeStoreSize(ScalarTy)));
+
+ auto ReplaceWithUnitStrided = [&]() {
+ VPBuilder Builder(VPI);
+ VPValue *StrideInBytes = Plan.getOrAddLiveIn(TypeSize->getValue());
+ auto *VecPtr =
+ new VPVectorPointerRecipe(PtrOp, ScalarTy, StrideInBytes,
+ GEPNoWrapFlags::none(), VPI->getDebugLoc());
+ Builder.insert(VecPtr);
+ if (VPI->getOpcode() == Instruction::Load) {
+ auto *WideLoad = new VPWidenLoadRecipe(
+ cast<LoadInst>(*VPI->getUnderlyingInstr()), VecPtr, VPI->getMask(),
+ true, *VPI, VPI->getDebugLoc());
+ Builder.insert(WideLoad);
+ VPI->replaceAllUsesWith(WideLoad);
+ } else {
+ auto *WideStore = new VPWidenStoreRecipe(
+ cast<StoreInst>(*VPI->getUnderlyingInstr()), VecPtr,
+ VPI->getOperand(0), VPI->getMask(), true, *VPI, VPI->getDebugLoc());
+ Builder.insert(WideStore);
+ }
+ VPI->eraseFromParent();
+ };
+
+ if (isa<SCEVConstant>(Stride)) {
+ if (Stride != TypeSize)
+ return Skip();
+
+ // Earlier MV helped with this memory operation too.
+ ReplaceWithUnitStrided();
+ return;
+ }
+
+ const SCEVConstant *StrideConstantMultiplier;
+ const SCEV *StrideNonConstantMultiplier;
+
+ const SCEV *ToMultiVersion = Stride;
+ const SCEV *MVConst = TypeSize;
+ if (match(Stride, m_scev_c_Mul(m_SCEVConstant(StrideConstantMultiplier),
+ m_SCEV(StrideNonConstantMultiplier)))) {
+ if (TypeSize != StrideConstantMultiplier) {
+ // TODO: Support `TypeSize = N * StrideCosntantMultiplier`,
+ // including negative `N`. For now, only process when they're equal,
+ // which matches the usefull part of the legacy behavior that
+ // multiversiones GEP index for stride one.
+ return Skip();
+ }
+ ToMultiVersion = StrideNonConstantMultiplier;
+ MVConst = SE->getOne(ToMultiVersion->getType());
+ } else if (!TypeSize->isOne()) {
+ // Likewise - try to match legacy behavior.
+ return Skip();
+ }
+
+ while (auto *C = dyn_cast<SCEVIntegralCastExpr>(ToMultiVersion)) {
+ ToMultiVersion = C->getOperand();
+ MVConst = SE->getTruncateOrSignExtend(MVConst, ToMultiVersion->getType());
+ }
+
+ if (!isa<SCEVUnknown>(ToMultiVersion)) {
+ // Match legacy behavior.
+ // If/when changed, make sure that explicit poison/undef in the defining
+ // expression doesn't cause any issues.
+ return Skip();
+ }
+
+ Value *StrideVal = cast<SCEVUnknown>(ToMultiVersion)->getValue();
+
+ if (isa<UndefValue>(StrideVal))
+ return Skip();
+
+ const SCEVPredicate *NewPred =
+ SE->getComparePredicate(CmpInst::ICMP_EQ, ToMultiVersion, MVConst);
+
+ // Check if new predicate implies that backedge is never taken. If so, there
+ // is no reason to multiversion for it.
+ SmallVector<const SCEVPredicate *> Preds{&CostCtx.PSE.getPredicate(),
+ &StridePredicates, NewPred};
+ auto *PredicatedMaxBTC = SE->rewriteUsingPredicate(
+ SE->getSymbolicMaxBackedgeTakenCount(CostCtx.L), CostCtx.L,
+ StridePredicates.getUnionWith(NewPred, *SE)
+ .getUnionWith(&CostCtx.PSE.getPredicate(), *SE));
+
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ return SE->isKnownPositive(SE->getMinusSCEV(
+ SE->getConstant(PredicatedMaxBTC->getType(),
+ VF.isScalable() ? 1 : VF.getFixedValue() - 1),
+ PredicatedMaxBTC));
+ },
+ Range))
+ return Skip();
+
+ StridePredicates = StridePredicates.getUnionWith(NewPred, *SE);
+
+ auto ReplaceMVUses = [&](Value *V) {
+ VPValue *From = Plan.getLiveIn(V);
+ if (!From)
+ return;
+ VPValue *To = Plan.getConstantInt(
+ From->getScalarType(),
+ cast<SCEVConstant>(MVConst)->getAPInt().getLimitedValue());
+ // TODO: Why is "If" necessary?
+ From->replaceUsesWithIf(To, [&](VPUser &U, unsigned) {
+ auto *R = cast<VPRecipeBase>(&U);
+ return R->getRegion() ||
+ R->getParent() ==
+ Plan.getVectorLoopRegion()->getSinglePredecessor();
+ });
+ };
+
+ ReplaceMVUses(StrideVal);
+ for (auto *U : StrideVal->users())
+ if (isa<SExtInst, ZExtInst>(U))
+ ReplaceMVUses(U);
+
+ ReplaceWithUnitStrided();
+ });
+
+ MemOps.swap(RemainingOps);
+
+ if (StridePredicates.isAlwaysTrue())
+ return;
+
+ VPBasicBlock *Entry = Plan.getEntry();
+ VPBuilder Builder(Entry);
+
+ auto *Pred = Builder.createExpandSCEVPredicate(StridePredicates);
+
+ auto *StridesCheckBB = Plan.createVPBasicBlock("strides.check");
+ VPBlockBase *ScalarPH = Plan.getScalarPreheader();
+ VPBlockUtils::insertBlockBefore(StridesCheckBB, Plan.getVectorPreheader());
+ VPBlockUtils::connectBlocks(StridesCheckBB, ScalarPH);
+ // SCEVExpander::expandCodeForPredicate would negate the condition, so scalar
+ // preheader should be the first successor.
+ std::swap(StridesCheckBB->getSuccessors()[0],
+ StridesCheckBB->getSuccessors()[1]);
+ Builder.setInsertPoint(StridesCheckBB);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, Pred);
+
+ for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
+ auto &Phi = cast<VPPhi>(R);
+ Phi.addIncoming(Phi.getIncomingValueForBlock(Entry));
+ }
+
+ auto RewriteVPExpandSCEV = [&](VPExpandSCEVRecipe *R) {
+ const SCEV *S = R->getSCEV();
+ Builder.setInsertPoint(R);
+ const SCEV *NewS =
+ SE->rewriteUsingPredicate(S, CostCtx.L, StridePredicates);
+ if (NewS == S)
+ return;
+ auto *NewR = Builder.createExpandSCEV(NewS);
+ R->replaceAllUsesWith(NewR);
+
+ // If this recipe is a trip count then we need to reset it explicitly.
+ if (R == Plan.getTripCount())
+ Plan.resetTripCount(NewR);
+
+ R->eraseFromParent();
+ };
+
+ if (auto *R = dyn_cast<VPExpandSCEVRecipe>(Plan.getTripCount())) {
+ RewriteVPExpandSCEV(R);
+ }
+
+ for (auto &R : make_early_inc_range(*Entry))
+ if (auto *ExpandSCEV = dyn_cast<VPExpandSCEVRecipe>(&R))
+ RewriteVPExpandSCEV(ExpandSCEV);
+}
+
void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) { return VF.isScalar(); }, Range))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85375625d34b5..5fe168e2dcc4c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -585,6 +585,12 @@ struct VPlanTransforms {
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
+ /// \p MemOps must be updated to contain ones that haven't been processed by
+ /// the pass.
+ static void multiversionForUnitStridedMemOps(
+ VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps);
+
/// Make VPlan-based scalarization decision prior to delegating to the ones
/// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
/// enabled by prior widening of consecutive memory operations for now.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 03e1ead89c169..98994291779b1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -426,8 +426,9 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
return Expr->isVectorToScalar();
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ // VPExpandSCEVRecipes and VPExpandStridePredicatesRecipe must be placed in
+ // the entry and are always uniform.
+ return isa<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>(VPV);
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index db4d2ce2738a2..206dceb238d3d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -disable-output \
-; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
-; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s
+; RUN: -vplan-print-after=multiversionForUnitStridedMemOps \
+; RUN: -enable-mem-access-versioning=false -enable-vplan-based-stride-mv 2>&1 | FileCheck %s
define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-LABEL: VPlan for loop in 'basic'
@@ -12,31 +12,38 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -45,7 +52,7 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -106,7 +113,8 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -115,7 +123,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -124,7 +132,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -187,7 +195,8 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -196,7 +205,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -205,7 +214,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -257,32 +266,39 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<8>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<1>, ir<8>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -291,7 +307,7 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -357,7 +373,8 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -366,7 +383,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -375,7 +392,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -442,7 +459,8 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -451,7 +469,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -460,7 +478,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -526,7 +544,8 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -535,7 +554,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -544,7 +563,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -608,7 +627,8 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -617,7 +637,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -626,7 +646,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -691,7 +711,8 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -700,7 +721,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -709,7 +730,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -758,34 +779,42 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -794,7 +823,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -847,35 +876,44 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride0 == 1
+; CHECK-NEXT: Equal predicate: %stride1 == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
-; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -884,7 +922,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -939,36 +977,43 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%stride1> = add ir<%stride>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%stride1> = add ir<1>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -977,7 +1022,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1036,36 +1081,43 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%stride0> = add ir<%stride>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride0> = add ir<1>, ir<1>
; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
-; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1074,7 +1126,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1151,7 +1203,8 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: EMIT-SCALAR ir<%ld1.ext> = sext ir<%ld1> to i64
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1160,7 +1213,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1169,7 +1222,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1245,7 +1298,8 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1254,7 +1308,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1263,7 +1317,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1322,34 +1376,42 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%gep.ld0>, ir<1>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1358,7 +1420,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1414,32 +1476,39 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld.base> = getelementptr ir<%p>, ir<%offset>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%gep.ld.base>, ir<%mul>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1448,7 +1517,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1500,32 +1569,39 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%mul>, ir<%offset>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1534,7 +1610,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1601,7 +1677,8 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1610,7 +1687,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1619,7 +1696,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1688,7 +1765,8 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1697,7 +1775,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1706,7 +1784,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1765,31 +1843,38 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %n)
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1798,7 +1883,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1862,7 +1947,8 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%stride>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1871,7 +1957,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1880,7 +1966,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1945,7 +2031,8 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1954,7 +2041,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1963,7 +2050,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2014,32 +2101,39 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %n = mul i64 %m, %stride
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (%stride * %m))
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %m)
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2048,7 +2142,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2097,32 +2191,39 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %n = add i64 %stride, 3
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV 4
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2131,7 +2232,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2213,35 +2314,43 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %p.out = getelementptr i8, ptr %p2, i64 %out.offset
; CHECK-NEXT: IR %n = add i64 %stride, 3
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV 4
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.ld2> = getelementptr ir<%p2>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld2> = load ir<%gep.ld2>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ld2>, ir<1>
+; CHECK-NEXT: WIDEN ir<%ld2> = load vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%ld2>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2250,7 +2359,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2322,7 +2431,8 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%base>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2331,7 +2441,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2340,7 +2450,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2391,31 +2501,38 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<42>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2424,7 +2541,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2488,7 +2605,8 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<42>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2497,7 +2615,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2506,7 +2624,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2568,7 +2686,8 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2577,7 +2696,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2586,7 +2705,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2633,32 +2752,39 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride.i32 == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT-SCALAR ir<%stride> = sext ir<%stride.i32> to i64
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = sext ir<1> to i64
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2667,7 +2793,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2716,32 +2842,39 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride.i64 == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<1> to i32
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<4>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2750,7 +2883,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2801,13 +2934,18 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %stride.trunc = trunc i32 %stride to i16
; CHECK-NEXT: IR %stride.ext = sext i32 %stride to i64
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
@@ -2815,23 +2953,26 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: EMIT-SCALAR ir<%iv.trunc> = trunc ir<%iv> to i16
; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
; CHECK-NEXT: EMIT ir<%idx.trunc> = mul ir<%iv.trunc>, ir<%stride.trunc>
-; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<%stride.ext>
+; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%p0>, ir<%idx.trunc>
; CHECK-NEXT: EMIT ir<%gep.ext> = getelementptr ir<%p0>, ir<%idx.ext>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld.trunc> = load ir<%gep.trunc>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld.ext> = load ir<%gep.ext>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.trunc>, ir<4>
+; CHECK-NEXT: WIDEN ir<%ld.trunc> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ext>, ir<4>
+; CHECK-NEXT: WIDEN ir<%ld.ext> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld.trunc>, ir<%ld.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2840,7 +2981,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3007,7 +3148,8 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -3016,7 +3158,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3025,7 +3167,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3072,31 +3214,38 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%idx>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<8>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3105,7 +3254,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3153,33 +3302,41 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%gep.ptr.st> = getelementptr ir<%p.ptr.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%gep.ld>, ir<%gep.ptr.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ptr.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3188,7 +3345,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3241,32 +3398,39 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3275,7 +3439,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3325,33 +3489,40 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%iv.times.stride>, ir<42>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3360,7 +3531,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3438,33 +3609,39 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %start.ext = zext i32 %start to i64
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV (zext i32 %start to i64)
-; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV (100 + (-1 * (zext i32 %start to i64))<nsw>)<nsw>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV 1
+; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV 99
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %start == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP5]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv.0> = WIDEN-INDUCTION ir<%start.ext>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv.0> = WIDEN-INDUCTION ir<1>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: ir<%iv.1> = WIDEN-INDUCTION ir<0>, vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv.1>
-; CHECK-NEXT: EMIT store ir<0.000000e+00>, ir<%gep.dst>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<4>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<0.000000e+00>
; CHECK-NEXT: EMIT ir<%iv.1.next> = add ir<%iv.1>, vp<[[VP3]]>
; CHECK-NEXT: EMIT ir<%iv.0.next> = add ir<%iv.0>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.0.next>, ir<100>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv.0>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%iv.1>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv.0>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = exiting-iv-value ir<%iv.1>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP4]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3473,8 +3650,8 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<%start.ext>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP8]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<%start.ext>, ir-bb<entry> ], [ ir<%start.ext>, strides.check ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP11]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 4429b0e731616..1c9931e3e881b 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -1,57 +1,53 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
-; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV
+; RUN: -enable-mem-access-versioning=false -enable-vplan-based-stride-mv 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
-; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
+; RUN: -enable-mem-access-versioning=false -enable-vplan-based-stride-mv 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @basic(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -137,7 +133,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -240,7 +236,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -318,80 +314,74 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @byte_gep_scaled_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 3)
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDE_X8:%.*]] = mul i64 [[STRIDE]], 8
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE_X8]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @byte_gep_scaled_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 3)
+; COMPARE-LAA-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
-; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[STRIDE_X8:%.*]] = mul i64 [[STRIDE]], 8
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE_X8]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -455,7 +445,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -495,7 +485,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -562,7 +552,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -602,7 +592,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -668,7 +658,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -708,7 +698,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -772,7 +762,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -783,18 +773,20 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
@@ -804,26 +796,15 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -886,7 +867,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -960,61 +941,50 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @shared_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP3]]
; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP10]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @shared_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1080,85 +1050,70 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; COMPARE-NO-MV-LABEL: define void @independent_strides(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE0]], 1
+; COMPARE-NO-MV-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE1]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE0]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE1]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP6]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP8]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP12]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP14]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP16]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP3]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP19]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP15]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> poison, i64 [[TMP26]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP30]], i64 [[TMP27]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = add <4 x i64> [[TMP25]], [[TMP33]]
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP34]], ptr [[TMP35]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE0]]
+; COMPARE-NO-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE1]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @independent_strides(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE0]], 1
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE1]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP0]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1211,41 +1166,26 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-NO-MV-LABEL: define void @dependent_strides(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP7]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP14]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load <4 x i64>, ptr [[TMP14]], align 8
; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP15]], align 8
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP16]], align 8
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
@@ -1255,23 +1195,40 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 2
; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 [[TMP30]], i32 3
; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDE1:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE1]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @dependent_strides(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1360,67 +1317,69 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-NO-MV-LABEL: define void @dependent_strides_reverse_order(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP7]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP7]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP14]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 3
; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP15]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP16]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP14]], align 8
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> poison, i64 [[TMP27]], i32 0
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 1
; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 2
; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 [[TMP30]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP34]], [[WIDE_LOAD]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP10]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDE0:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE0]]
+; COMPARE-NO-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @dependent_strides_reverse_order(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1552,7 +1511,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1563,18 +1522,20 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP3]]
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
@@ -1598,30 +1559,15 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP23]], i32 3
; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = sext <4 x i32> [[TMP27]] to <4 x i64>
; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP15]], [[TMP28]]
-; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
-; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1701,7 +1647,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1712,18 +1658,20 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP3]]
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
@@ -1747,30 +1695,15 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP28]], [[TMP16]]
-; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
-; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
-; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1808,61 +1741,50 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @strided_interleave(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP5]], i61 1
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP6]], i61 1
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP7]], i61 1
; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP8]], i61 1
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP10]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[GEP_LD0]], i61 1
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @strided_interleave(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1877,7 +1799,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1894,7 +1816,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1931,49 +1853,47 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-NO-MV-LABEL: define void @in_loop_base(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[MUL:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = add i64 [[MUL]], [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD_BASE:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[GEP_LD_BASE]], i64 [[MUL]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @in_loop_base(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
@@ -1986,7 +1906,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2002,7 +1922,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2036,51 +1956,46 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-NO-MV-LABEL: define void @base_not_in_ir(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[OFFSET]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add <4 x i64> [[TMP0]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], [[OFFSET]]
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[MUL:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = add i64 [[MUL]], [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @base_not_in_ir(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -2093,7 +2008,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2108,7 +2023,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2173,7 +2088,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2214,7 +2129,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2282,7 +2197,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2324,7 +2239,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2365,46 +2280,30 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
@@ -2415,18 +2314,18 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
@@ -2440,7 +2339,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2456,7 +2355,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2522,7 +2421,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2538,16 +2437,56 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2555,7 +2494,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2622,7 +2561,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2638,7 +2577,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2646,9 +2585,49 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2656,7 +2635,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2691,46 +2670,30 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
@@ -2741,7 +2704,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2749,12 +2712,11 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX1]], 4
@@ -2768,7 +2730,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2784,7 +2746,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2815,51 +2777,24 @@ exit:
define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2867,7 +2802,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2875,10 +2810,9 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -2899,7 +2833,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2955,31 +2889,12 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
-; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2987,7 +2902,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3013,62 +2928,35 @@ header:
exit:
ret void
-}
-
-; Same as above but memdep check doesn't depend on stride
-define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
-; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
-; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+}
+
+; Same as above but memdep check doesn't depend on stride
+define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
+; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P2]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[P_OUT]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
@@ -3082,7 +2970,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3091,10 +2979,9 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
@@ -3125,7 +3012,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3196,7 +3083,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3239,7 +3126,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -3275,48 +3162,44 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @nd_array_last_idx(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @nd_array_last_idx(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -3328,7 +3211,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3342,7 +3225,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3405,7 +3288,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3416,18 +3299,20 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[INDEX]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP1]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP2]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP3]], i64 42
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
@@ -3437,26 +3322,15 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[IDX]], i64 42
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP44:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -3516,7 +3390,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3588,49 +3462,45 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-NO-MV-LABEL: define void @sext_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[STRIDE_I32:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STRIDE_I32]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sext <4 x i32> [[BROADCAST_SPLAT]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = sext i32 [[STRIDE_I32]] to i64
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP47:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @sext_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[STRIDE_I32:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -3689,8 +3559,11 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-LABEL: define void @trunc_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK1:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK1]]:
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = sub i32 0, [[TMP0]]
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 0
@@ -3703,44 +3576,25 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 [[TMP5]]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_I64]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i32>
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = mul <4 x i32> [[VEC_IND]], [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP10]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP11]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP10]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP12]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP10]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP13]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP10]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP14]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load <4 x i32>, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[TMP14]]
; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP26]], ptr [[TMP27]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP14]], 4
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK1]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
@@ -3749,21 +3603,35 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-NO-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP49:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @trunc_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = trunc i64 [[STRIDE_I64]] to i32
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = sub i32 0, [[TMP9]]
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP9]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 [[TMP9]]
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP3]], i32 127)
+; COMPARE-LAA-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; COMPARE-LAA-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = sub i32 0, [[MUL_RESULT]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp slt i32 [[MUL_RESULT]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = select i1 [[TMP11]], i1 [[TMP6]], i1 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
@@ -3774,9 +3642,10 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
@@ -3819,8 +3688,11 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i32 [[STRIDE]] to i16
; COMPARE-NO-MV-NEXT: [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK1:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK1]]:
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub i16 0, [[STRIDE_TRUNC]]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = icmp slt i16 [[STRIDE_TRUNC]], 0
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i16 [[TMP0]], i16 [[STRIDE_TRUNC]]
@@ -3832,66 +3704,31 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp sgt i16 [[TMP3]], 0
; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = select i1 [[TMP1]], i1 [[TMP5]], i1 [[TMP4]]
; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[MUL_OVERFLOW]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[STRIDE_TRUNC]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_EXT]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND3:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = sext <4 x i32> [[VEC_IND]] to <4 x i64>
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul <4 x i16> [[VEC_IND3]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = mul <4 x i64> [[TMP8]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[TMP9]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i16> [[TMP9]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP11]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i16> [[TMP9]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP12]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = extractelement <4 x i16> [[TMP9]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = trunc i32 [[INDEX]] to i16
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = sext i32 [[INDEX]] to i64
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = mul i16 [[TMP8]], [[STRIDE_TRUNC]]
; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP13]]
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP14]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP15]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP14]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP16]]
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP14]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP17]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP14]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP18]]
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP19]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP20]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP21]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP22]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP23]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP24]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP25]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP26]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> poison, i32 [[TMP35]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP37]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP38]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP42:%.*]] = load <4 x i32>, ptr [[TMP26]], align 4
; COMPARE-NO-MV-NEXT: [[TMP43:%.*]] = add <4 x i32> [[TMP34]], [[TMP42]]
; COMPARE-NO-MV-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP43]], ptr [[TMP44]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT4]] = add <4 x i16> [[VEC_IND3]], splat (i16 4)
; COMPARE-NO-MV-NEXT: [[TMP45:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK1]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-NO-MV-NEXT: [[IV_TRUNC:%.*]] = trunc i32 [[IV]] to i16
; COMPARE-NO-MV-NEXT: [[IV_EXT:%.*]] = sext i32 [[IV]] to i64
@@ -3905,7 +3742,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-NO-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP51:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3914,14 +3751,27 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i32 [[STRIDE]] to i16
; COMPARE-LAA-MV-NEXT: [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = sub i16 0, [[STRIDE_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = icmp slt i16 [[STRIDE_TRUNC]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i16 [[TMP8]], i16 [[STRIDE_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[TMP10]], i16 127)
+; COMPARE-LAA-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
+; COMPARE-LAA-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = sub i16 0, [[MUL_RESULT]]
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = icmp slt i16 [[MUL_RESULT]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = icmp sgt i16 [[TMP11]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = select i1 [[TMP9]], i1 [[TMP13]], i1 [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP15]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = mul i16 [[TMP0]], [[STRIDE_TRUNC]]
@@ -3938,9 +3788,10 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IV_TRUNC:%.*]] = trunc i32 [[IV]] to i16
; COMPARE-LAA-MV-NEXT: [[IV_EXT:%.*]] = sext i32 [[IV]] to i64
@@ -4055,7 +3906,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP6]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4066,79 +3917,66 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = icmp sge <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP22]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE]]:
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF1]]:
-; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = add i64 [[TMP2]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP22]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP7]]
+; COMPARE-LAA-MV-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP9]], ptr [[TMP10]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE2]]:
; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF3]]:
-; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = add i64 [[TMP2]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP22]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP12]]
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP25]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP14]], ptr [[TMP15]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE4]]:
; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; COMPARE-LAA-MV: [[PRED_STORE_IF5]]:
-; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = add i64 [[TMP2]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP22]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
+; COMPARE-LAA-MV-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP23]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE6]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[C:%.*]] = icmp sge i64 [[IV]], [[X]]
-; COMPARE-LAA-MV-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[LATCH]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
; COMPARE-LAA-MV: [[IF]]:
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[LATCH]]
+; COMPARE-LAA-MV-NEXT: br label %[[LATCH:.*]]
; COMPARE-LAA-MV: [[LATCH]]:
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP53:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -4201,7 +4039,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4238,7 +4076,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -4271,48 +4109,44 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @basic_strided_store(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 0
-; COMPARE-NO-MV-NEXT: store i64 [[TMP6]], ptr [[TMP10]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 1
-; COMPARE-NO-MV-NEXT: store i64 [[TMP7]], ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 2
-; COMPARE-NO-MV-NEXT: store i64 [[TMP8]], ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 3
-; COMPARE-NO-MV-NEXT: store i64 [[TMP9]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP55:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic_strided_store(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4324,7 +4158,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4338,7 +4172,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP55:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4370,47 +4204,51 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-NO-MV-LABEL: define void @ptr_vec_use(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP2]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP3]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP7]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> [[TMP11]], i64 [[TMP8]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP14]], align 8
; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x ptr> [[TMP1]], ptr [[TMP15]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP57:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @ptr_vec_use(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4427,7 +4265,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4443,7 +4281,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP57:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4478,49 +4316,48 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @stride_idx_vec_use(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = mul <4 x i64> [[TMP16]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = mul i64 [[LD]], [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_idx_vec_use(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4535,7 +4372,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4550,7 +4387,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4583,50 +4420,51 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-LABEL: define void @offset_stride_idx_vec_use(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add <4 x i64> [[TMP0]], splat (i64 42)
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP6]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[TMP17]], [[TMP1]]
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IV_TIMES_STRIDE:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = add i64 [[IV_TIMES_STRIDE]], 42
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = mul i64 [[LD]], [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @offset_stride_idx_vec_use(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4643,7 +4481,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4659,7 +4497,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4755,43 +4593,26 @@ exit:
define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-LABEL: define void @test_rewrite_iv_scevs(
; COMPARE-NO-MV-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub nsw i64 100, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ne i32 [[START]], 1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = mul i64 1, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = mul i64 2, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], [[TMP7]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul i64 3, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[DST]], i64 [[OFFSET_IDX]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP6]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP11]], align 4
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP12]], align 4
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP13]], align 4
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP14]], align 4
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-NO-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP14]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 4
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; COMPARE-NO-MV-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 97, %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[STRIDES_CHECK]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[STRIDES_CHECK]] ]
; COMPARE-NO-MV-NEXT: br label %[[LOOP:.*]]
; COMPARE-NO-MV: [[LOOP]]:
; COMPARE-NO-MV-NEXT: [[IV_0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_0_NEXT:%.*]], %[[LOOP]] ]
@@ -4801,7 +4622,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -4809,10 +4630,9 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 100, [[START_EXT]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[START]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[START]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4822,7 +4642,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 4
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4837,7 +4657,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-LAA-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP64:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
>From 9e7830d8a506466acf312539ce6ccf0df37d0f74 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 16 Jul 2026 16:54:18 -0700
Subject: [PATCH 04/13] Address review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 43 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.h | 9 ++--
3 files changed, 24 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 118d4402cbf37..b3e261597c9cf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6741,7 +6741,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
- RecipeBuilder, CostCtx);
+ RecipeBuilder, CostCtx, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6d556a3bd29be..3895520253732 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7348,7 +7348,8 @@ static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx) {
+ VPCostContext &CostCtx,
+ VFSelectionContext &Config) {
// Collect all loads/stores first. We will start with ones having simpler
// decisions followed by more complex ones that are potentially
// guided/dependent on the simpler ones.
@@ -7494,7 +7495,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
if (EnableVPlanBasedStrideMV)
RUN_VPLAN_PASS(VPlanTransforms::multiversionForUnitStridedMemOps, Plan,
- CostCtx, RecipeBuilder, Range, MemOps);
+ CostCtx, Config, RecipeBuilder, Range, MemOps);
VPlanTransforms::runPass("delegateMemOpWideningToLegacyCM", ProcessSubset,
Plan, [&](VPInstruction *VPI) {
@@ -7507,8 +7508,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
}
void VPlanTransforms::multiversionForUnitStridedMemOps(
- VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
- VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps) {
+ VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
+ VPRecipeBuilder &RecipeBuilder, VFRange &Range,
+ SmallVectorImpl<VPInstruction *> &MemOps) {
if (CostCtx.L->getHeader()->getParent()->hasOptSize())
return;
SmallVector<VPInstruction *> RemainingOps;
@@ -7529,38 +7531,28 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
const SCEV *PtrSCEV =
vputils::getSCEVExprForVPValue(PtrOp, CostCtx.PSE, CostCtx.L);
- const SCEV *Start = nullptr;
- const SCEV *Stride = nullptr;
+ const SCEV *Start, *Stride;
if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
- m_SpecificLoop(CostCtx.L)))) {
+ m_SpecificLoop(CostCtx.L))))
return Skip();
- }
Type *ScalarTy = VPI->getOpcode() == Instruction::Load
? VPI->getScalarType()
: VPI->getOperand(0)->getScalarType();
if (VPI->getMask()) {
- auto &TTI = CostCtx.TTI;
Instruction *I = VPI->getUnderlyingInstr();
- unsigned AS = getLoadStoreAddressSpace(I);
- const Align Alignment = getLoadStoreAlignment(I);
if (!LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) -> bool {
- Type *VTy = VectorType::get(ScalarTy, VF);
- return VPI->getOpcode() == Instruction::Load
- ? (TTI.isLegalMaskedLoad(VTy, Alignment, AS) ||
- TTI.isLegalMaskedGather(VTy, Alignment))
- : (TTI.isLegalMaskedStore(VTy, Alignment, AS) ||
- TTI.isLegalMaskedScatter(VTy, Alignment));
+ return Config.isLegalMaskedLoadOrStore(I, VF);
},
Range))
return Skip();
}
const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
- Stride->getType(), SE->getDataLayout().getTypeStoreSize(ScalarTy)));
+ Stride->getType(), SE->getDataLayout().getTypeAllocSize(ScalarTy)));
auto ReplaceWithUnitStrided = [&]() {
VPBuilder Builder(VPI);
@@ -7619,6 +7611,9 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
MVConst = SE->getTruncateOrSignExtend(MVConst, ToMultiVersion->getType());
}
+ if (match(ToMultiVersion, m_scev_UndefOrPoison()))
+ return Skip();
+
if (!isa<SCEVUnknown>(ToMultiVersion)) {
// Match legacy behavior.
// If/when changed, make sure that explicit poison/undef in the defining
@@ -7628,16 +7623,11 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
Value *StrideVal = cast<SCEVUnknown>(ToMultiVersion)->getValue();
- if (isa<UndefValue>(StrideVal))
- return Skip();
-
const SCEVPredicate *NewPred =
SE->getComparePredicate(CmpInst::ICMP_EQ, ToMultiVersion, MVConst);
// Check if new predicate implies that backedge is never taken. If so, there
// is no reason to multiversion for it.
- SmallVector<const SCEVPredicate *> Preds{&CostCtx.PSE.getPredicate(),
- &StridePredicates, NewPred};
auto *PredicatedMaxBTC = SE->rewriteUsingPredicate(
SE->getSymbolicMaxBackedgeTakenCount(CostCtx.L), CostCtx.L,
StridePredicates.getUnionWith(NewPred, *SE)
@@ -7645,10 +7635,11 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
- return SE->isKnownPositive(SE->getMinusSCEV(
+ return SE->isKnownPredicate(
+ CmpPredicate(ICmpInst::ICMP_SLT), PredicatedMaxBTC,
SE->getConstant(PredicatedMaxBTC->getType(),
- VF.isScalable() ? 1 : VF.getFixedValue() - 1),
- PredicatedMaxBTC));
+ VF.isScalable() ? 1
+ : VF.getFixedValue() - 1));
},
Range))
return Skip();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 5fe168e2dcc4c..50be723a47405 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -13,6 +13,7 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
+#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
@@ -583,13 +584,15 @@ struct VPlanTransforms {
/// recipes. Non load/store input instructions are left unchanged.
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx);
+ VPCostContext &CostCtx,
+ VFSelectionContext &Config);
/// \p MemOps must be updated to contain ones that haven't been processed by
/// the pass.
static void multiversionForUnitStridedMemOps(
- VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
- VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps);
+ VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
+ VPRecipeBuilder &RecipeBuilder, VFRange &Range,
+ SmallVectorImpl<VPInstruction *> &MemOps);
/// Make VPlan-based scalarization decision prior to delegating to the ones
/// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
>From 8258e553e421f0e731c5f2b0a7965254d3f534d0 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 17 Jul 2026 09:18:10 -0700
Subject: [PATCH 05/13] Expand to VPlan recipes instead of to LLVM IR
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 42 +++++++++----------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 36 ++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 5 +++
.../LoopVectorize/vplan-based-stride-mv.ll | 8 ++--
4 files changed, 65 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3895520253732..68e9d080cb713 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5806,23 +5806,30 @@ void VPlanTransforms::expandSCEVsToVPInstructions(VPlan &Plan,
->getDebugLoc();
VPSCEVExpander Expander(Builder, SE, DL);
- // Expand VPExpandSCEVRecipes to VPInstructions using VPSCEVExpander. During
- // the transition, unsupported VPExpandSCEVRecipes are skipped and left for
- // late expansion.
+ // Expand VPExpandSCEVRecipes and VPExpandStridePredicatesRecipes to
+ // VPInstructions using VPSCEVExpander. During the transition, unsupported
+ // recipes are skipped and left for late expansion.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpSCEV || ExpSCEV->user_empty())
+ auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
+ if (!Def || Def->user_empty())
+ continue;
+
+ auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(Def);
+ auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Def);
+ if (!ExpStrides && !ExpSCEV)
continue;
- Builder.setInsertPoint(ExpSCEV);
- VPValue *Expanded = Expander.tryToExpand(ExpSCEV->getSCEV());
+ Builder.setInsertPoint(Def);
+ VPValue *Expanded =
+ ExpSCEV ? Expander.tryToExpand(ExpSCEV->getSCEV())
+ : Expander.tryToExpandPredicate(ExpStrides->getSCEVPredicate());
if (!Expanded)
continue;
- ExpSCEV->replaceAllUsesWith(Expanded);
+ Def->replaceAllUsesWith(Expanded);
// TripCount should not be used after expansion to VPInstructions. Reset to
// poison to avoid dangling references.
- if (Plan.getTripCount() == ExpSCEV)
- Plan.resetTripCount(Plan.getPoison(ExpSCEV->getScalarType()));
- ExpSCEV->eraseFromParent();
+ if (Plan.getTripCount() == Def)
+ Plan.resetTripCount(Plan.getPoison(Def->getScalarType()));
+ Def->eraseFromParent();
}
}
@@ -5835,17 +5842,6 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
DenseMap<const SCEV *, Value *> ExpandedSCEVs;
// Expand remaining VPExpandSCEVRecipes to IR instructions using SCEVExpander.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- if (auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(&R)) {
- Value *Res = Expander.expandCodeForPredicate(
- ExpStrides->getSCEVPredicate(), EntryBB->getTerminator());
- Res->setName("strides.mv.check");
- VPValue *Exp = Plan.getOrAddLiveIn(Res);
-
- ExpStrides->replaceAllUsesWith(Exp);
- ExpStrides->eraseFromParent();
- continue;
- }
-
auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
if (!ExpSCEV)
continue;
@@ -5859,6 +5855,8 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
Plan.resetTripCount(Exp);
ExpSCEV->eraseFromParent();
}
+ // NOTE: All current uses of VPExpandStridePredicatesRecipe are expected to be
+ // expandable via VPSCEVExpander, thus no handling here.
assert(none_of(*Entry,
IsaPred<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>) &&
"all VPExpandSCEVRecipes/VPExpandStridePredicatesRecipe must have "
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 98994291779b1..1d92873883e3d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -893,3 +893,39 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
return nullptr;
}
}
+
+VPValue *VPSCEVExpander::tryToExpandPredicate(const SCEVPredicate *Pred) {
+ if (auto *CmpPred = dyn_cast<SCEVComparePredicate>(Pred)) {
+ VPValue *LHS = tryToExpand(CmpPred->getLHS());
+ if (!LHS)
+ return nullptr;
+ VPValue *RHS = tryToExpand(CmpPred->getRHS());
+ if (!RHS)
+ return nullptr;
+ // Match SCEVExpander::expandComparePredicate behavior:
+ auto InvPred = CmpInst::getInversePredicate(CmpPred->getPredicate());
+ return Builder.createICmp(InvPred, LHS, RHS, DL);
+ }
+
+ if (auto *UnionPred = dyn_cast<SCEVUnionPredicate>(Pred)) {
+ SmallVector<VPValue *, 4> Checks;
+ for (const auto *Pred : UnionPred->getPredicates()) {
+ VPValue *Check = tryToExpandPredicate(Pred);
+ if (!Check)
+ return nullptr;
+ Checks.push_back(Check);
+ }
+
+ if (Checks.empty())
+ return Builder.getPlan().getOrAddLiveIn(
+ ConstantInt::getFalse(Builder.getPlan().getContext()));
+
+ VPValue *Result = Checks[0];
+ for (VPValue *Check : ArrayRef<VPValue *>(Checks).drop_front())
+ Result = Builder.createOr(Result, Check, DL);
+ return Result;
+ }
+
+ assert(false && "We shouldn't be generating that kind of predicate!");
+ return nullptr;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2980b704ec8da..649efaadda163 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -189,6 +189,11 @@ class VPSCEVExpander {
/// Try to expand \p S into recipes and live-ins using the builder. Returns
/// nullptr if \p S cannot be expanded yet.
VPValue *tryToExpand(const SCEV *S);
+
+ /// Try to expand \p Pred into recipes and live-ins using the builder. Returns
+ /// nullptr if \p Pred cannot be expanded (not expected to happen for the
+ /// predicates we currently generate).
+ VPValue *tryToExpandPredicate(const SCEVPredicate *Pred);
};
//===----------------------------------------------------------------------===//
// Utilities for modifying predecessors and successors of VPlan blocks.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 1c9931e3e881b..6fccb632640fa 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -2280,8 +2280,8 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
@@ -2321,8 +2321,8 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
@@ -2670,8 +2670,8 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
@@ -2712,8 +2712,8 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
>From 6e1e9a07941d596593e5a33a18ca2a80f4150158 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Jul 2026 10:58:04 -0700
Subject: [PATCH 06/13] Address code-review comments
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 ++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 8 +-
.../VPlan/vplan-based-stride-mv.ll | 118 +-----------------
3 files changed, 14 insertions(+), 127 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 68e9d080cb713..48ed60e0489e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7491,7 +7491,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
return ReplaceWith(VPI, WidenedR);
});
- if (EnableVPlanBasedStrideMV)
+ if (EnableVPlanBasedStrideMV &&
+ !CostCtx.L->getHeader()->getParent()->hasOptSize())
RUN_VPLAN_PASS(VPlanTransforms::multiversionForUnitStridedMemOps, Plan,
CostCtx, Config, RecipeBuilder, Range, MemOps);
@@ -7509,8 +7510,6 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
VPRecipeBuilder &RecipeBuilder, VFRange &Range,
SmallVectorImpl<VPInstruction *> &MemOps) {
- if (CostCtx.L->getHeader()->getParent()->hasOptSize())
- return;
SmallVector<VPInstruction *> RemainingOps;
ScalarEvolution *SE = CostCtx.PSE.getSE();
@@ -7591,9 +7590,9 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (match(Stride, m_scev_c_Mul(m_SCEVConstant(StrideConstantMultiplier),
m_SCEV(StrideNonConstantMultiplier)))) {
if (TypeSize != StrideConstantMultiplier) {
- // TODO: Support `TypeSize = N * StrideCosntantMultiplier`,
+ // TODO: Support `TypeSize = N * StrideConstantMultiplier`,
// including negative `N`. For now, only process when they're equal,
- // which matches the usefull part of the legacy behavior that
+ // which matches the useful part of the legacy behavior that
// multiversiones GEP index for stride one.
return Skip();
}
@@ -7634,7 +7633,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
return SE->isKnownPredicate(
- CmpPredicate(ICmpInst::ICMP_SLT), PredicatedMaxBTC,
+ ICmpInst::ICMP_SLT, PredicatedMaxBTC,
SE->getConstant(PredicatedMaxBTC->getType(),
VF.isScalable() ? 1
: VF.getFixedValue() - 1));
@@ -7679,7 +7678,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
auto *Pred = Builder.createExpandSCEVPredicate(StridePredicates);
auto *StridesCheckBB = Plan.createVPBasicBlock("strides.check");
- VPBlockBase *ScalarPH = Plan.getScalarPreheader();
+ VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
VPBlockUtils::insertBlockBefore(StridesCheckBB, Plan.getVectorPreheader());
VPBlockUtils::connectBlocks(StridesCheckBB, ScalarPH);
// SCEVExpander::expandCodeForPredicate would negate the condition, so scalar
@@ -7689,7 +7688,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
Builder.setInsertPoint(StridesCheckBB);
Builder.createNaryOp(VPInstruction::BranchOnCond, Pred);
- for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
+ for (VPRecipeBase &R : ScalarPH->phis()) {
auto &Phi = cast<VPPhi>(R);
Phi.addIncoming(Phi.getIncomingValueForBlock(Entry));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 1d92873883e3d..3ba822f1d6ed8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -917,15 +917,13 @@ VPValue *VPSCEVExpander::tryToExpandPredicate(const SCEVPredicate *Pred) {
}
if (Checks.empty())
- return Builder.getPlan().getOrAddLiveIn(
- ConstantInt::getFalse(Builder.getPlan().getContext()));
+ return Builder.getPlan().getFalse();
VPValue *Result = Checks[0];
- for (VPValue *Check : ArrayRef<VPValue *>(Checks).drop_front())
+ for (VPValue *Check : drop_begin(Checks))
Result = Builder.createOr(Result, Check, DL);
return Result;
}
- assert(false && "We shouldn't be generating that kind of predicate!");
- return nullptr;
+ llvm_unreachable("Unhandled predicate");
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index 206dceb238d3d..42d405036b98a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -89,64 +89,9 @@ exit:
ret void
}
+; Unit-stride speculation isn't even attempted, so no VPlan dumps.
+; CHECK-NOT: @basic_optsize
define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
-; CHECK-LABEL: VPlan for loop in 'basic_optsize'
-; CHECK: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in ir<128> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
-; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: Successor(s): ir-bb<header>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<header>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
-; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
-; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
-; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
-; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
-; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
-; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-;
entry:
br label %header
@@ -171,64 +116,9 @@ exit:
attributes #0 = { optsize }
+; Unit-stride speculation isn't even attempted, so no VPlan dumps.
+; CHECK-NOT: @basic_minsize
define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
-; CHECK-LABEL: VPlan for loop in 'basic_minsize'
-; CHECK: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in ir<128> = original trip-count
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
-; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
-; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
-; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: Successor(s): ir-bb<header>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<header>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ] (extra operand: vp<%bc.resume.val> from scalar.ph)
-; CHECK-NEXT: IR %iv.next = add nsw i64 %iv, 1
-; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
-; CHECK-NEXT: IR %gep.ld = getelementptr i64, ptr %p, i64 %idx
-; CHECK-NEXT: IR %ld = load i64, ptr %gep.ld, align 8
-; CHECK-NEXT: IR %gep.st = getelementptr i64, ptr %p.out, i64 %iv
-; CHECK-NEXT: IR store i64 %ld, ptr %gep.st, align 8
-; CHECK-NEXT: IR %exitcond = icmp slt i64 %iv.next, 128
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-;
entry:
br label %header
>From 770d4b72121c6a464116cd1cf8a0e68854dd8c38 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 20 Jul 2026 11:22:20 -0700
Subject: [PATCH 07/13] Expand directly to VPlan, drop
VPExpandStridePredicatesRecipe
---
.../Vectorize/LoopVectorizationPlanner.h | 6 --
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 -----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 --
.../Transforms/Vectorize/VPlanTransforms.cpp | 50 ++++++--------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 5 +-
.../VPlan/vplan-based-stride-mv.ll | 69 ++++++++++---------
7 files changed, 60 insertions(+), 123 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f6c5da8126c7f..cd02ced2942d1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -516,12 +516,6 @@ class VPBuilder {
return tryInsertInstruction(new VPExpandSCEVRecipe(Expr));
}
- VPExpandStridePredicatesRecipe *
- createExpandSCEVPredicate(const SCEVUnionPredicate &StridePredicates) {
- return tryInsertInstruction(new VPExpandStridePredicatesRecipe(
- StridePredicates, this->getPlan().getContext()));
- }
-
VPVectorPointerRecipe *
createVectorPointer(VPValue *Ptr, Type *SourceElementTy, VPValue *Stride,
GEPNoWrapFlags GEPFlags, DebugLoc DL) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 10f957609f238..7d69b3453cba9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -677,9 +677,7 @@ void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
auto RecipeIndent = Indent + " ";
for (const VPRecipeBase &Recipe : *this) {
Recipe.print(O, RecipeIndent, SlotTracker);
- // SCEVPredicate::print adds a newline so we don't want to add one for it:
- if (!isa<VPExpandStridePredicatesRecipe>(Recipe))
- O << '\n';
+ O << '\n';
}
printSuccessors(O, Indent);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 64ab410a77b53..b67d1a1281c14 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -418,7 +418,6 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPBranchOnMaskSC,
VPDerivedIVSC,
VPExpandSCEVSC,
- VPExpandStridePredicatesSC,
VPExpressionSC,
VPIRInstructionSC,
VPInstructionSC,
@@ -626,7 +625,6 @@ class LLVM_ABI_FOR_TEST VPSingleDefRecipe : public VPRecipeBase,
switch (R->getVPRecipeID()) {
case VPRecipeBase::VPDerivedIVSC:
case VPRecipeBase::VPExpandSCEVSC:
- case VPRecipeBase::VPExpandStridePredicatesSC:
case VPRecipeBase::VPExpressionSC:
case VPRecipeBase::VPInstructionSC:
case VPRecipeBase::VPReductionEVLSC:
@@ -4023,46 +4021,6 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
#endif
};
-class VPExpandStridePredicatesRecipe : public VPSingleDefRecipe {
- SCEVUnionPredicate StridePredicates;
-
-public:
- VPExpandStridePredicatesRecipe(const SCEVUnionPredicate &StridePredicates,
- LLVMContext &Ctx)
- : VPSingleDefRecipe(VPRecipeBase::VPExpandStridePredicatesSC, {},
- Type::getInt1Ty(Ctx)),
- StridePredicates(StridePredicates) {}
-
- ~VPExpandStridePredicatesRecipe() override = default;
-
- VPExpandStridePredicatesRecipe *clone() override {
- return new VPExpandStridePredicatesRecipe(StridePredicates,
- getScalarType()->getContext());
- }
-
- VP_CLASSOF_IMPL(VPRecipeBase::VPExpandStridePredicatesSC)
-
- void execute(VPTransformState &State) override {
- llvm_unreachable("SCEVPredicates must be expanded before final execute");
- }
-
- /// Return the cost of this VPExpandSCEVRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override {
- // TODO: Compute accurate cost after retiring the legacy cost model.
- return 0;
- }
-
- const SCEVPredicate *getSCEVPredicate() const { return &StridePredicates; }
-
-protected:
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-};
-
/// A recipe for generating the active lane mask for the vector loop that is
/// used to predicate the vector operations.
class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 75fb9b3734e94..c62c4a082b95e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -4824,13 +4824,6 @@ void VPExpandSCEVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
printAsOperand(O, SlotTracker);
O << " = EXPAND SCEV " << *Expr;
}
-
-void VPExpandStridePredicatesRecipe::printRecipe(
- raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const {
- O << Indent << "EMIT ";
- printAsOperand(O, SlotTracker);
- O << " = EXPAND SCEVPredicate " << StridePredicates;
-}
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 48ed60e0489e9..3f04da479125e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5806,30 +5806,23 @@ void VPlanTransforms::expandSCEVsToVPInstructions(VPlan &Plan,
->getDebugLoc();
VPSCEVExpander Expander(Builder, SE, DL);
- // Expand VPExpandSCEVRecipes and VPExpandStridePredicatesRecipes to
- // VPInstructions using VPSCEVExpander. During the transition, unsupported
- // recipes are skipped and left for late expansion.
+ // Expand VPExpandSCEVRecipes to VPInstructions using VPSCEVExpander. During
+ // the transition, unsupported VPExpandSCEVRecipes are skipped and left for
+ // late expansion.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
- if (!Def || Def->user_empty())
- continue;
-
- auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(Def);
- auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Def);
- if (!ExpStrides && !ExpSCEV)
+ auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
+ if (!ExpSCEV || ExpSCEV->user_empty())
continue;
- Builder.setInsertPoint(Def);
- VPValue *Expanded =
- ExpSCEV ? Expander.tryToExpand(ExpSCEV->getSCEV())
- : Expander.tryToExpandPredicate(ExpStrides->getSCEVPredicate());
+ Builder.setInsertPoint(ExpSCEV);
+ VPValue *Expanded = Expander.tryToExpand(ExpSCEV->getSCEV());
if (!Expanded)
continue;
- Def->replaceAllUsesWith(Expanded);
+ ExpSCEV->replaceAllUsesWith(Expanded);
// TripCount should not be used after expansion to VPInstructions. Reset to
// poison to avoid dangling references.
- if (Plan.getTripCount() == Def)
- Plan.resetTripCount(Plan.getPoison(Def->getScalarType()));
- Def->eraseFromParent();
+ if (Plan.getTripCount() == ExpSCEV)
+ Plan.resetTripCount(Plan.getPoison(ExpSCEV->getScalarType()));
+ ExpSCEV->eraseFromParent();
}
}
@@ -5855,12 +5848,8 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
Plan.resetTripCount(Exp);
ExpSCEV->eraseFromParent();
}
- // NOTE: All current uses of VPExpandStridePredicatesRecipe are expected to be
- // expandable via VPSCEVExpander, thus no handling here.
- assert(none_of(*Entry,
- IsaPred<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>) &&
- "all VPExpandSCEVRecipes/VPExpandStridePredicatesRecipe must have "
- "been expanded");
+ assert(none_of(*Entry, IsaPred<VPExpandSCEVRecipe>) &&
+ "all VPExpandSCEVRecipes must have been expanded");
// Add IR instructions in the entry basic block but not in the VPIRBasicBlock
// to the VPIRBasicBlock.
auto EI = Entry->begin();
@@ -7674,15 +7663,20 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
VPBasicBlock *Entry = Plan.getEntry();
VPBuilder Builder(Entry);
-
- auto *Pred = Builder.createExpandSCEVPredicate(StridePredicates);
+ DebugLoc DL = cast<VPIRBasicBlock>(Entry)
+ ->getIRBasicBlock()
+ ->getTerminator()
+ ->getDebugLoc();
+ VPSCEVExpander Expander(Builder, *SE, DL);
+ VPValue *Pred = Expander.tryToExpandPredicate(&StridePredicates);
+ assert(Pred && "Must be expandable!");
auto *StridesCheckBB = Plan.createVPBasicBlock("strides.check");
VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
VPBlockUtils::insertBlockBefore(StridesCheckBB, Plan.getVectorPreheader());
VPBlockUtils::connectBlocks(StridesCheckBB, ScalarPH);
- // SCEVExpander::expandCodeForPredicate would negate the condition, so scalar
- // preheader should be the first successor.
+ // SCEVExpander/VPSCEVExpander::expandCodeForPredicate negate the condition,
+ // so scalar preheader should be the first successor.
std::swap(StridesCheckBB->getSuccessors()[0],
StridesCheckBB->getSuccessors()[1]);
Builder.setInsertPoint(StridesCheckBB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 3ba822f1d6ed8..115813222abb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -426,9 +426,8 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
return Expr->isVectorToScalar();
- // VPExpandSCEVRecipes and VPExpandStridePredicatesRecipe must be placed in
- // the entry and are always uniform.
- return isa<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>(VPV);
+ // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
+ return isa<VPExpandSCEVRecipe>(VPV);
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index 42d405036b98a..e6510158a7efc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -12,7 +12,7 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -156,7 +156,7 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -669,7 +669,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -766,19 +766,20 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride0 == 1
-; CHECK-NEXT: Equal predicate: %stride1 == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp ne ir<%stride1>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = or vp<[[VP3]]>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP5]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
@@ -787,23 +788,23 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
-; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
-; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP9]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP10]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -812,7 +813,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP12]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -867,7 +868,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -971,7 +972,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -1266,7 +1267,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -1366,7 +1367,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -1459,7 +1460,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -1733,7 +1734,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %n)
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -1992,7 +1993,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %n = mul i64 %m, %stride
; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %m)
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -2082,7 +2083,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %n = add i64 %stride, 3
; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV 4
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -2205,7 +2206,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: IR %p.out = getelementptr i8, ptr %p2, i64 %out.offset
; CHECK-NEXT: IR %n = add i64 %stride, 3
; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV 4
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -2391,7 +2392,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -2642,7 +2643,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride.i32 == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride.i32>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -2732,7 +2733,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride.i64 == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride.i64>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -2824,7 +2825,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %stride.trunc = trunc i32 %stride to i16
; CHECK-NEXT: IR %stride.ext = sext i32 %stride to i64
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -3104,7 +3105,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -3192,7 +3193,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -3288,7 +3289,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -3379,7 +3380,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp ne ir<%stride>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
@@ -3501,7 +3502,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; CHECK-NEXT: IR %start.ext = zext i32 %start to i64
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV 1
; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV 99
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %start == 1
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = icmp ne ir<%start>, ir<1>
; CHECK-NEXT: Successor(s): scalar.ph, strides.check
; CHECK-EMPTY:
; CHECK-NEXT: strides.check:
>From 4c8cde5fdd74513dff2ed3c853684a16c1ed2278 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 21 Jul 2026 10:24:44 -0700
Subject: [PATCH 08/13] Use erase_if/drop Skip
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 31 +++++++------------
1 file changed, 12 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3f04da479125e..260ed8a9913f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7499,19 +7499,13 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
VPRecipeBuilder &RecipeBuilder, VFRange &Range,
SmallVectorImpl<VPInstruction *> &MemOps) {
- SmallVector<VPInstruction *> RemainingOps;
-
ScalarEvolution *SE = CostCtx.PSE.getSE();
-
PredicatedScalarEvolution StrideMVPSE(*SE, const_cast<Loop &>(*CostCtx.L));
-
SCEVUnionPredicate StridePredicates({}, *SE);
- // Use `for_each` so that we could do `return Skip();`.
- for_each(MemOps, [&](VPInstruction *VPI) {
- auto Skip = [&]() { RemainingOps.push_back(VPI); };
+ erase_if(MemOps, [&](VPInstruction *VPI) -> bool {
if (RecipeBuilder.isConsecutiveWithoutVPlanBasedStrideSpeculation(VPI))
- return Skip();
+ return false;
auto *PtrOp = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
@@ -7521,7 +7515,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
m_SpecificLoop(CostCtx.L))))
- return Skip();
+ return false;
Type *ScalarTy = VPI->getOpcode() == Instruction::Load
? VPI->getScalarType()
@@ -7534,7 +7528,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
return Config.isLegalMaskedLoadOrStore(I, VF);
},
Range))
- return Skip();
+ return false;
}
const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
@@ -7564,11 +7558,11 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (isa<SCEVConstant>(Stride)) {
if (Stride != TypeSize)
- return Skip();
+ return false;
// Earlier MV helped with this memory operation too.
ReplaceWithUnitStrided();
- return;
+ return true;
}
const SCEVConstant *StrideConstantMultiplier;
@@ -7583,13 +7577,13 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
// including negative `N`. For now, only process when they're equal,
// which matches the useful part of the legacy behavior that
// multiversiones GEP index for stride one.
- return Skip();
+ return false;
}
ToMultiVersion = StrideNonConstantMultiplier;
MVConst = SE->getOne(ToMultiVersion->getType());
} else if (!TypeSize->isOne()) {
// Likewise - try to match legacy behavior.
- return Skip();
+ return false;
}
while (auto *C = dyn_cast<SCEVIntegralCastExpr>(ToMultiVersion)) {
@@ -7598,13 +7592,13 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
}
if (match(ToMultiVersion, m_scev_UndefOrPoison()))
- return Skip();
+ return false;
if (!isa<SCEVUnknown>(ToMultiVersion)) {
// Match legacy behavior.
// If/when changed, make sure that explicit poison/undef in the defining
// expression doesn't cause any issues.
- return Skip();
+ return false;
}
Value *StrideVal = cast<SCEVUnknown>(ToMultiVersion)->getValue();
@@ -7628,7 +7622,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
: VF.getFixedValue() - 1));
},
Range))
- return Skip();
+ return false;
StridePredicates = StridePredicates.getUnionWith(NewPred, *SE);
@@ -7654,10 +7648,9 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
ReplaceMVUses(U);
ReplaceWithUnitStrided();
+ return true;
});
- MemOps.swap(RemainingOps);
-
if (StridePredicates.isAlwaysTrue())
return;
>From f2a41d5c72423446b8bb1e310823aebc942e9d61 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 21 Jul 2026 11:29:40 -0700
Subject: [PATCH 09/13] Address code review comments
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 260ed8a9913f5..301db57e97cb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7616,7 +7616,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
return SE->isKnownPredicate(
- ICmpInst::ICMP_SLT, PredicatedMaxBTC,
+ ICmpInst::ICMP_ULT, PredicatedMaxBTC,
SE->getConstant(PredicatedMaxBTC->getType(),
VF.isScalable() ? 1
: VF.getFixedValue() - 1));
@@ -7633,7 +7633,9 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
VPValue *To = Plan.getConstantInt(
From->getScalarType(),
cast<SCEVConstant>(MVConst)->getAPInt().getLimitedValue());
- // TODO: Why is "If" necessary?
+
+ // Original scalar loop can still use `From`, make sure to only rewrite
+ // uses inside the vector loop that we guard with the checks.
From->replaceUsesWithIf(To, [&](VPUser &U, unsigned) {
auto *R = cast<VPRecipeBase>(&U);
return R->getRegion() ||
@@ -7697,13 +7699,12 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
R->eraseFromParent();
};
- if (auto *R = dyn_cast<VPExpandSCEVRecipe>(Plan.getTripCount())) {
- RewriteVPExpandSCEV(R);
- }
-
for (auto &R : make_early_inc_range(*Entry))
if (auto *ExpandSCEV = dyn_cast<VPExpandSCEVRecipe>(&R))
RewriteVPExpandSCEV(ExpandSCEV);
+
+ if (auto *R = dyn_cast<VPExpandSCEVRecipe>(Plan.getTripCount()))
+ RewriteVPExpandSCEV(R);
}
void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
>From f3424c771cf1073e0a96500e9bd0dc164cc05557 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 22 Jul 2026 13:36:21 -0700
Subject: [PATCH 10/13] Inline `RewriteVPExpandSCEV` lambda
Calling it on `Plan.getTripCount()` explicitly doesn't seem to be
necessary anymore, so can inline into the single remaining use.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 25 ++++++++-----------
1 file changed, 11 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 301db57e97cb3..792aa2df8193d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7682,29 +7682,26 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
Phi.addIncoming(Phi.getIncomingValueForBlock(Entry));
}
- auto RewriteVPExpandSCEV = [&](VPExpandSCEVRecipe *R) {
- const SCEV *S = R->getSCEV();
- Builder.setInsertPoint(R);
+ for (auto &R : make_early_inc_range(*Entry)) {
+ auto *ExpandSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
+ if (!ExpandSCEV)
+ continue;
+
+ const SCEV *S = ExpandSCEV->getSCEV();
+ Builder.setInsertPoint(ExpandSCEV);
const SCEV *NewS =
SE->rewriteUsingPredicate(S, CostCtx.L, StridePredicates);
if (NewS == S)
return;
auto *NewR = Builder.createExpandSCEV(NewS);
- R->replaceAllUsesWith(NewR);
+ ExpandSCEV->replaceAllUsesWith(NewR);
// If this recipe is a trip count then we need to reset it explicitly.
- if (R == Plan.getTripCount())
+ if (ExpandSCEV == Plan.getTripCount())
Plan.resetTripCount(NewR);
- R->eraseFromParent();
- };
-
- for (auto &R : make_early_inc_range(*Entry))
- if (auto *ExpandSCEV = dyn_cast<VPExpandSCEVRecipe>(&R))
- RewriteVPExpandSCEV(ExpandSCEV);
-
- if (auto *R = dyn_cast<VPExpandSCEVRecipe>(Plan.getTripCount()))
- RewriteVPExpandSCEV(R);
+ ExpandSCEV->eraseFromParent();
+ }
}
void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
>From 894392dec23d7a64e2f7db1e0ec25e232aa8786a Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Mon, 27 Jul 2026 11:53:35 -0700
Subject: [PATCH 11/13] Outline widenUnitStridedLoadStore helper
Reuses it for both normal and speculative unit-strided memaccesses widening.
Apparently, I misread `createVectorPointer` interface (or maybe it changed
throughout the life of this PR), so this also fixes a bug in the new code.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 99 +++++++++----------
.../VPlan/vplan-based-stride-mv.ll | 52 +++++-----
2 files changed, 70 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ffac2431ce902..c0918542d4a0c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -62,6 +62,46 @@ static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
return getStrideFromAddRec(AddRec, L, AccessTy, /*Ptr=*/nullptr, PSE);
}
+/// Replace unit-strided load/store recipe \p VPI by a widened consecutive
+/// memory recipe. If \p Reverse is true, adjust pointer creation, masks and
+/// values to match reverse access order.
+static void widenUnitStridedLoadStore(VPInstruction *VPI, bool Reverse,
+ bool IsPredicated) {
+ VPBuilder Builder(VPI);
+ Instruction *I = VPI->getUnderlyingInstr();
+ bool IsLoad = VPI->getOpcode() == Instruction::Load;
+ VPValue *Ptr = VPI->getOperand(!IsLoad);
+ Type *ScalarTy =
+ IsLoad ? VPI->getScalarType() : VPI->getOperand(0)->getScalarType();
+ VPValue *VectorPtr = Builder.createConsecutiveVectorPointer(
+ Ptr, ScalarTy, Reverse, VPI->getDebugLoc());
+ VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
+ if (Reverse && Mask)
+ Mask =
+ Builder.createNaryOp(VPInstruction::Reverse, Mask, VPI->getDebugLoc());
+
+ if (IsLoad) {
+ VPSingleDefRecipe *Load =
+ Builder.createWidenLoad(*cast<LoadInst>(I), VectorPtr, Mask,
+ /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
+ if (Reverse)
+ Load = Builder.createNaryOp(VPInstruction::Reverse, Load,
+ VPI->getDebugLoc());
+ VPI->replaceAllUsesWith(Load);
+ VPI->eraseFromParent();
+ return;
+ }
+
+ VPValue *StoredVal = VPI->getOperand(0);
+ if (Reverse)
+ StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
+ VPI->getDebugLoc());
+
+ Builder.createWidenStore(*cast<StoreInst>(I), VectorPtr, StoredVal, Mask,
+ /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
+ VPI->eraseFromParent();
+}
+
bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
Loop *OuterLoop) {
@@ -5487,37 +5527,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
getLoadStoreAddressSpace(I)))
return false;
- VPBuilder Builder(VPI);
- VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
- Ptr, ScalarTy, Reverse, VPI->getDebugLoc());
-
- VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
- // Reverse the mask so it matches the reversed access order.
- if (Reverse && Mask)
- Mask = Builder.createNaryOp(VPInstruction::Reverse, Mask,
- VPI->getDebugLoc());
-
- if (IsLoad) {
- VPSingleDefRecipe *Load = Builder.createWidenLoad(
- *cast<LoadInst>(I), VectorPtr, Mask,
- /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
- // Reverse the loaded values back into program order.
- if (Reverse)
- Load = Builder.createNaryOp(VPInstruction::Reverse, Load,
- VPI->getDebugLoc());
- return ReplaceWith(VPI, Load);
- }
-
- VPValue *StoredVal = VPI->getOperand(0);
- if (Reverse)
- // Reverse the stored values so they are written in descending order.
- StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
- VPI->getDebugLoc());
-
- auto *StoreR = Builder.createWidenStore(
- *cast<StoreInst>(I), VectorPtr, StoredVal, Mask,
- /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
- return ReplaceWith(VPI, StoreR);
+ widenUnitStridedLoadStore(VPI, Reverse, IsPredicated);
+ return true;
});
if (EnableVPlanBasedStrideMV &&
@@ -5576,34 +5587,12 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
Stride->getType(), SE->getDataLayout().getTypeAllocSize(ScalarTy)));
- auto ReplaceWithUnitStrided = [&]() {
- VPBuilder Builder(VPI);
- VPValue *StrideInBytes = Plan.getOrAddLiveIn(TypeSize->getValue());
- auto *VecPtr =
- new VPVectorPointerRecipe(PtrOp, ScalarTy, StrideInBytes,
- GEPNoWrapFlags::none(), VPI->getDebugLoc());
- Builder.insert(VecPtr);
- if (VPI->getOpcode() == Instruction::Load) {
- auto *WideLoad = new VPWidenLoadRecipe(
- cast<LoadInst>(*VPI->getUnderlyingInstr()), VecPtr, VPI->getMask(),
- true, *VPI, VPI->getDebugLoc());
- Builder.insert(WideLoad);
- VPI->replaceAllUsesWith(WideLoad);
- } else {
- auto *WideStore = new VPWidenStoreRecipe(
- cast<StoreInst>(*VPI->getUnderlyingInstr()), VecPtr,
- VPI->getOperand(0), VPI->getMask(), true, *VPI, VPI->getDebugLoc());
- Builder.insert(WideStore);
- }
- VPI->eraseFromParent();
- };
-
if (isa<SCEVConstant>(Stride)) {
if (Stride != TypeSize)
return false;
// Earlier MV helped with this memory operation too.
- ReplaceWithUnitStrided();
+ widenUnitStridedLoadStore(VPI, /*Reverse=*/false, VPI->getMask());
return true;
}
@@ -5691,7 +5680,7 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (isa<SExtInst, ZExtInst>(U))
ReplaceMVUses(U);
- ReplaceWithUnitStrided();
+ widenUnitStridedLoadStore(VPI, /*Reverse=*/false, VPI->getMask());
return true;
});
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index e6510158a7efc..39dd63f0ff152 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -30,7 +30,7 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -175,7 +175,7 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<1>, ir<8>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -688,9 +688,9 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -788,9 +788,9 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP8]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP9]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -889,7 +889,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
@@ -994,7 +994,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -1286,9 +1286,9 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%gep.ld0>, ir<1>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -1386,7 +1386,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld.base> = getelementptr ir<%p>, ir<%offset>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%gep.ld.base>, ir<%mul>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -1479,7 +1479,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%mul>, ir<%offset>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -1752,7 +1752,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -2011,7 +2011,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -2101,7 +2101,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -2224,7 +2224,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.ld2> = getelementptr ir<%p2>, ir<%iv>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ld2>, ir<1>
@@ -2410,7 +2410,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<42>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -2662,7 +2662,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -2752,7 +2752,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<4>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -2847,9 +2847,9 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%p0>, ir<%idx.trunc>
; CHECK-NEXT: EMIT ir<%gep.ext> = getelementptr ir<%p0>, ir<%idx.ext>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.trunc>, ir<4>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.trunc>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld.trunc> = load vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ext>, ir<4>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ext>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld.ext> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld.trunc>, ir<%ld.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -3126,7 +3126,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%idx>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<8>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
@@ -3211,7 +3211,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
@@ -3307,7 +3307,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -3399,7 +3399,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%iv.times.stride>, ir<42>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
@@ -3519,7 +3519,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; CHECK-NEXT: ir<%iv.0> = WIDEN-INDUCTION ir<1>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: ir<%iv.1> = WIDEN-INDUCTION ir<0>, vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv.1>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<4>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<0.000000e+00>
; CHECK-NEXT: EMIT ir<%iv.1.next> = add ir<%iv.1>, vp<[[VP3]]>
; CHECK-NEXT: EMIT ir<%iv.0.next> = add ir<%iv.0>, ir<1>
>From de737fcc50fcd132d2b9b3c03a8d21f9e3819b3b Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 28 Jul 2026 15:25:32 -0700
Subject: [PATCH 12/13] Drop `isConsecutiveWithoutVPlanBasedStrideSpeculation`
Not necessary after all unit-strided memops are widened on VPlan prior
to this pass.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 ----------
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 2 --
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 --
3 files changed, 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ed30b6f7b1ae6..aa099b84ed61b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6393,16 +6393,6 @@ bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
return false;
}
-bool VPRecipeBuilder::isConsecutiveWithoutVPlanBasedStrideSpeculation(
- VPInstruction *MemOp) {
- auto *I = MemOp->getUnderlyingInstr();
- auto *PtrOp = getLoadStorePointerOperand(I);
- auto *ScalarTy = MemOp->getOpcode() == Instruction::Load
- ? I->getType()
- : I->getOperand(0)->getType();
- return Legal->isConsecutivePtr(ScalarTy, PtrOp);
-}
-
VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
VFRange &Range) {
auto *I = VPI->getUnderlyingInstr();
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 0281192d68f3b..54b05136a45b5 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -90,8 +90,6 @@ class VPRecipeBuilder {
/// predicated, add the mask as last operand. Range.End may be decreased to
/// ensure same recipe behavior from \p Range.Start to \p Range.End.
VPSingleDefRecipe *handleReplication(VPInstruction *VPI, VFRange &Range);
-
- bool isConsecutiveWithoutVPlanBasedStrideSpeculation(VPInstruction *MemOp);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f374638fdded5..f706e00215407 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5550,8 +5550,6 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
SCEVUnionPredicate StridePredicates({}, *SE);
erase_if(MemOps, [&](VPInstruction *VPI) -> bool {
- if (RecipeBuilder.isConsecutiveWithoutVPlanBasedStrideSpeculation(VPI))
- return false;
auto *PtrOp = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
: VPI->getOperand(1);
>From 9c8bba85fc3e3a155cf80bcc72beba01117fd6b7 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 29 Jul 2026 16:48:21 -0700
Subject: [PATCH 13/13] Revert remaining change in VPRecipeBuilder.h
---
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 54b05136a45b5..1303b62d5faf4 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -33,6 +33,11 @@ class VPRecipeBuilder {
VPBuilder &Builder;
+ /// Check if \p I can be widened at the start of \p Range and possibly
+ /// decrease the range such that the returned value holds for the entire \p
+ /// Range. The function should not be called for memory instructions or calls.
+ bool shouldWiden(Instruction *I, VFRange &Range) const;
+
/// Optimize the special case where the operand of \p VPI is a constant
/// integer induction variable.
VPWidenIntOrFpInductionRecipe *
@@ -56,11 +61,6 @@ class VPRecipeBuilder {
/// Returns true if the target prefers vectorized addressing.
bool prefersVectorizedAddressing() const;
- /// Check if \p I can be widened at the start of \p Range and possibly
- /// decrease the range such that the returned value holds for the entire \p
- /// Range. The function should not be called for memory instructions or calls.
- bool shouldWiden(Instruction *I, VFRange &Range) const;
-
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
More information about the llvm-commits
mailing list