[llvm] [NFC][VPlan] Add extra tests for VPlan-based stride speculation (PR #218795)
Andrei Elovikov via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 08:06:00 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/218795
>From 6db058f6a84fd37d44afaf446169832087fcbf02 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 25 Aug 2026 15:10:16 -0700
Subject: [PATCH 1/2] [NFC][VPlan] Add extra tests for VPlan-based stride
speculation
---
.../VPlan/vplan-based-stride-mv.ll | 925 +++++++++--
.../vplan-based-stride-mv-btc.ll | 582 +++----
.../LoopVectorize/vplan-based-stride-mv.ll | 1396 ++++++++++++-----
3 files changed, 2108 insertions(+), 795 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index ea44f400bf4a3..45687811d808a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -352,6 +352,84 @@ exit:
ret void
}
+; Like @byte_gep_under_scaled_stride, but with an extra-narrow use of %stride
+; in the loop to exercise MV replacement through `trunc ... to i1`.
+define void @byte_gep_under_scaled_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'byte_gep_under_scaled_stride_extra_narrow_use'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride.i1 = trunc i64 %stride to i1
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<4>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%ld>, ir<2>
+; CHECK-NEXT: EMIT ir<%val> = select ir<%stride.i1>, ir<%ld>, ir<%mul>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+;
+entry:
+ %stride.i1 = trunc i64 %stride to i1
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 4
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %mul = mul i64 %ld, 2
+ %val = select i1 %stride.i1, i64 %ld, i64 %mul
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
; Another variation for constant multiplier with byte gep. This time the
; multiplier is bigger than load access type so this cannot be speculated for
; unit-strideness.
@@ -2288,9 +2366,10 @@ exit:
ret void
}
-; Stride is used through `trunc` in the loop.
-define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
-; CHECK-LABEL: VPlan for loop in 'trunc_stride'
+; Stride "i1 1" becomes "-1" when performing `sext`. However, it's better not to
+; multiversion/speculate it to begin with.
+define void @i1_zext_stride_with_sext_use(ptr noalias %p.out, ptr %p, i1 %stride) {
+; CHECK-LABEL: VPlan for loop in 'i1_zext_stride_with_sext_use'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -2298,6 +2377,8 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride.zext = zext i1 %stride to i64
+; CHECK-NEXT: IR %stride.sext = sext i1 %stride to i32
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
@@ -2308,13 +2389,13 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.zext>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%stride.sext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2336,31 +2417,34 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: Successor(s): ir-bb<header>
;
entry:
+ %stride.zext = zext i1 %stride to i64
+ %stride.sext = sext i1 %stride to i32
br label %header
header:
- %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
- %stride = trunc i64 %stride.i64 to i32
- %iv.next = add nsw i32 %iv, 1
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
- %idx = mul i32 %iv, %stride
+ %idx = mul i64 %iv, %stride.zext
- %gep.ld = getelementptr i32, ptr %p, i32 %idx
- %ld = load i32, ptr %gep.ld, align 8
+ %gep.ld = getelementptr i32, ptr %p, i64 %idx
+ %ld = load i32, ptr %gep.ld, align 4
- %gep.st = getelementptr i32, ptr %p.out, i32 %iv
- store i32 %ld, ptr %gep.st, align 8
+ %val = add i32 %ld, %stride.sext
- %exitcond = icmp slt i32 %iv.next, 128
+ %gep.st = getelementptr i32, ptr %p.out, i64 %iv
+ store i32 %val, ptr %gep.st, align 4
+
+ %exitcond = icmp slt i64 %iv.next, 128
br i1 %exitcond, label %header, label %exit
exit:
ret void
}
-; %stride is usedc through both `trunc`/`sext` for different accesses.
-define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride) {
-; CHECK-LABEL: VPlan for loop in 'trunc_ext_stride'
+; Similar to above, but "real" stride is never positive.
+define void @i1_sext_stride_with_zext_use(ptr noalias %p.out, ptr %p, i1 %stride) {
+; CHECK-LABEL: VPlan for loop in 'i1_sext_stride_with_zext_use'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -2368,8 +2452,8 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %stride.trunc = trunc i32 %stride to i16
-; CHECK-NEXT: IR %stride.ext = sext i32 %stride to i64
+; CHECK-NEXT: IR %stride.zext = zext i1 %stride to i32
+; CHECK-NEXT: IR %stride.sext = sext i1 %stride to i64
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
@@ -2381,15 +2465,10 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT-SCALAR ir<%iv.trunc> = trunc ir<%iv> to i16
-; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
-; CHECK-NEXT: EMIT ir<%idx.trunc> = mul ir<%iv.trunc>, ir<%stride.trunc>
-; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<%stride.ext>
-; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%p0>, ir<%idx.trunc>
-; CHECK-NEXT: EMIT ir<%gep.ext> = getelementptr ir<%p0>, ir<%idx.ext>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld.trunc> = load ir<%gep.trunc>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld.ext> = load ir<%gep.ext>
-; CHECK-NEXT: EMIT ir<%val> = add ir<%ld.trunc>, ir<%ld.ext>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.sext>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%stride.zext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
@@ -2413,42 +2492,34 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: Successor(s): ir-bb<header>
;
entry:
- %stride.trunc = trunc i32 %stride to i16
- %stride.ext = sext i32 %stride to i64
+ %stride.zext = zext i1 %stride to i32
+ %stride.sext = sext i1 %stride to i64
br label %header
header:
- %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i32 %iv, 1
-
- %iv.trunc = trunc i32 %iv to i16
- %iv.ext = sext i32 %iv to i64
-
- %idx.trunc = mul i16 %iv.trunc, %stride.trunc
- %idx.ext = mul i64 %iv.ext, %stride.ext
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
- %gep.trunc = getelementptr i32, ptr %p0, i16 %idx.trunc
- %gep.ext = getelementptr i32, ptr %p0, i64 %idx.ext
+ %idx = mul i64 %iv, %stride.sext
- %ld.trunc = load i32, ptr %gep.trunc, align 4
- %ld.ext = load i32, ptr %gep.ext, align 4
+ %gep.ld = getelementptr i32, ptr %p, i64 %idx
+ %ld = load i32, ptr %gep.ld, align 4
- %val = add i32 %ld.trunc, %ld.ext
+ %val = add i32 %ld, %stride.zext
- %gep.st = getelementptr i32, ptr %p.out, i32 %iv
- store i32 %val, ptr %gep.st, align 8
+ %gep.st = getelementptr i32, ptr %p.out, i64 %iv
+ store i32 %val, ptr %gep.st, align 4
- %exitcond = icmp slt i32 %iv.next, 128
+ %exitcond = icmp slt i64 %iv.next, 128
br i1 %exitcond, label %header, label %exit
exit:
ret void
}
-; Check that we don't speculate unit-strided masked memory access if masked wide
-; memory operation isn't legal (or that we properly pass the mask if it is).
-define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
-; CHECK-LABEL: VPlan for loop in 'basic_masked'
+; Stride is used through `trunc` in the loop.
+define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
+; CHECK-LABEL: VPlan for loop in 'trunc_stride'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -2466,19 +2537,13 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%c> = icmp sge ir<%iv>, ir<%x>
-; CHECK-NEXT: Successor(s): if
-; CHECK-EMPTY:
-; CHECK-NEXT: if:
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>, ir<%c>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>, ir<%c>
-; CHECK-NEXT: Successor(s): latch
-; CHECK-EMPTY:
-; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2503,32 +2568,27 @@ entry:
br label %header
header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
- %iv.next = add nsw i64 %iv, 1
- %c = icmp sge i64 %iv, %x
- br i1 %c, label %if, label %latch
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = trunc i64 %stride.i64 to i32
+ %iv.next = add nsw i32 %iv, 1
-if:
- %idx = mul i64 %iv, %stride
+ %idx = mul i32 %iv, %stride
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
+ %gep.ld = getelementptr i32, ptr %p, i32 %idx
+ %ld = load i32, ptr %gep.ld, align 8
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
- br label %latch
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %ld, ptr %gep.st, align 8
-latch:
- %exitcond = icmp slt i64 %iv.next, 128
+ %exitcond = icmp slt i32 %iv.next, 128
br i1 %exitcond, label %header, label %exit
exit:
ret void
}
-; See https://github.com/llvm/llvm-project/issues/162922.
-define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
-; CHECK-LABEL: VPlan for loop in 'stride_poison'
+define void @trunc_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
+; CHECK-LABEL: VPlan for loop in 'trunc_stride_extra_narrow_use'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -2536,6 +2596,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride.i1 = trunc i64 %stride.i64 to i1
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
@@ -2546,12 +2607,16 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
+; CHECK-NEXT: EMIT ir<%stride.byte> = mul ir<%stride>, ir<4>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<poison>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.byte>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%ld>, ir<2>
+; CHECK-NEXT: EMIT ir<%val> = select ir<%stride.i1>, ir<%ld>, ir<%mul>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2573,30 +2638,36 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): ir-bb<header>
;
entry:
+ %stride.i1 = trunc i64 %stride.i64 to i1
br label %header
header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = trunc i64 %stride.i64 to i32
+ %stride.byte = mul i32 %stride, 4
+ %iv.next = add nsw i32 %iv, 1
- %idx = mul i64 %iv, poison
+ %idx = mul i32 %iv, %stride.byte
- %gep.ld = getelementptr i64, ptr %p, i64 %idx
- %ld = load i64, ptr %gep.ld, align 8
+ %gep.ld = getelementptr i8, ptr %p, i32 %idx
+ %ld = load i32, ptr %gep.ld, align 8
- %gep.st = getelementptr i64, ptr %p.out, i64 %iv
- store i64 %ld, ptr %gep.st, align 8
+ %mul = mul i32 %ld, 2
+ %val = select i1 %stride.i1, i32 %ld, i32 %mul
- %exitcond = icmp slt i64 %iv.next, 128
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
br i1 %exitcond, label %header, label %exit
exit:
ret void
}
-; Tests above all used loads, make sure that store is supported too.
-define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
-; CHECK-LABEL: VPlan for loop in 'basic_strided_store'
+; %stride is used through both `trunc`/`sext` for different accesses.
+define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride) {
+; CHECK-LABEL: VPlan for loop in 'trunc_ext_stride'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -2604,6 +2675,8 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride.trunc = trunc i32 %stride to i16
+; CHECK-NEXT: IR %stride.ext = sext i32 %stride to i64
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
@@ -2615,11 +2688,17 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
-; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
-; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%idx>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv.trunc> = trunc ir<%iv> to i16
+; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
+; CHECK-NEXT: EMIT ir<%idx.trunc> = mul ir<%iv.trunc>, ir<%stride.trunc>
+; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<%stride.ext>
+; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%p0>, ir<%idx.trunc>
+; CHECK-NEXT: EMIT ir<%gep.ext> = getelementptr ir<%p0>, ir<%idx.ext>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld.trunc> = load ir<%gep.trunc>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld.ext> = load ir<%gep.ext>
+; CHECK-NEXT: EMIT ir<%val> = add ir<%ld.trunc>, ir<%ld.ext>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2641,31 +2720,41 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): ir-bb<header>
;
entry:
+ %stride.trunc = trunc i32 %stride to i16
+ %stride.ext = sext i32 %stride to i64
br label %header
header:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
- %iv.next = add nsw i64 %iv, 1
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i32 %iv, 1
- %idx = mul i64 %iv, %stride
+ %iv.trunc = trunc i32 %iv to i16
+ %iv.ext = sext i32 %iv to i64
- %gep.ld = getelementptr i64, ptr %p, i64 %iv
- %ld = load i64, ptr %gep.ld, align 8
+ %idx.trunc = mul i16 %iv.trunc, %stride.trunc
+ %idx.ext = mul i64 %iv.ext, %stride.ext
- %gep.st = getelementptr i64, ptr %p.out, i64 %idx
- store i64 %ld, ptr %gep.st, align 8
+ %gep.trunc = getelementptr i32, ptr %p0, i16 %idx.trunc
+ %gep.ext = getelementptr i32, ptr %p0, i64 %idx.ext
- %exitcond = icmp slt i64 %iv.next, 128
+ %ld.trunc = load i32, ptr %gep.trunc, align 4
+ %ld.ext = load i32, ptr %gep.ext, align 4
+
+ %val = add i32 %ld.trunc, %ld.ext
+
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
br i1 %exitcond, label %header, label %exit
exit:
ret void
}
-; This test shows how/if we scalarize address computation def-chain if that
-; pointer has other non-scalar uses.
-define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64 %stride) {
-; CHECK-LABEL: VPlan for loop in 'ptr_vec_use'
+; Stride is used through a chain of casts defined outside VPlan.
+define void @trunc_ext_chained_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'trunc_ext_chained_stride'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
@@ -2673,6 +2762,8 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride.trunc = trunc i64 %stride to i1
+; CHECK-NEXT: IR %stride.sext = sext i1 %stride.trunc to i64
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
@@ -2684,13 +2775,11 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.sext>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
-; CHECK-NEXT: EMIT ir<%gep.ptr.st> = getelementptr ir<%p.ptr.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%gep.ld>, ir<%gep.ptr.st>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2712,13 +2801,15 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: Successor(s): ir-bb<header>
;
entry:
+ %stride.trunc = trunc i64 %stride to i1
+ %stride.sext = sext i1 %stride.trunc to i64
br label %header
header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
%iv.next = add nsw i64 %iv, 1
- %idx = mul i64 %iv, %stride
+ %idx = mul i64 %iv, %stride.sext
%gep.ld = getelementptr i64, ptr %p, i64 %idx
%ld = load i64, ptr %gep.ld, align 8
@@ -2726,9 +2817,6 @@ header:
%gep.st = getelementptr i64, ptr %p.out, i64 %iv
store i64 %ld, ptr %gep.st, align 8
- %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
- store ptr %gep.ld, ptr %gep.ptr.st
-
%exitcond = icmp slt i64 %iv.next, 128
br i1 %exitcond, label %header, label %exit
@@ -2736,11 +2824,302 @@ exit:
ret void
}
-; Similar to above, but it's not the resulting pointer itself that has
-; non-scalar use but something in the middle of its def-chain.
-define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
-; CHECK-LABEL: VPlan for loop in 'stride_idx_vec_use'
-; CHECK: VPlan ' for UF>=1' {
+; Check that we don't speculate unit-strided masked memory access if masked wide
+; memory operation isn't legal (or that we properly pass the mask if it is).
+define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
+; CHECK-LABEL: VPlan for loop in 'basic_masked'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%c> = icmp sge ir<%iv>, ir<%x>
+; CHECK-NEXT: Successor(s): if
+; CHECK-EMPTY:
+; CHECK-NEXT: if:
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>, ir<%c>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>, ir<%c>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>, ir<%c>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %iv.next = add nsw i64 %iv, 1
+ %c = icmp sge i64 %iv, %x
+ br i1 %c, label %if, label %latch
+
+if:
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+ br label %latch
+
+latch:
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; See https://github.com/llvm/llvm-project/issues/162922.
+define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_poison'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<poison>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, poison
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Tests above all used loads, make sure that store is supported too.
+define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'basic_strided_store'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%idx>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %iv
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %idx
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; This test shows how/if we scalarize address computation def-chain if that
+; pointer has other non-scalar uses.
+define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'ptr_vec_use'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%gep.ptr.st> = getelementptr ir<%p.ptr.out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%gep.ld>, ir<%gep.ptr.st>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %gep.ptr.st = getelementptr ptr, ptr %p.ptr.out, i64 %iv
+ store ptr %gep.ld, ptr %gep.ptr.st
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Similar to above, but it's not the resulting pointer itself that has
+; non-scalar use but something in the middle of its def-chain.
+define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_idx_vec_use'
+; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
@@ -2985,4 +3364,312 @@ exit:
ret void
}
+; Used to crash because `getSymbolicMaxBackedgeTakenCount` is only available in
+; PSE but not SE.
+define void @stride_mv_predicated_btc(ptr noalias %p.out, ptr %p, i32 %M, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'stride_mv_predicated_btc'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (0 smax %M))<nuw>
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv.ext>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv.ext>
+; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv.next.ext> = sext ir<%iv.next> to i32
+; CHECK-NEXT: EMIT ir<%ec> = icmp sgt ir<%iv.next.ext>, ir<%M>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<header>
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i16 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.ext = sext i16 %iv to i64
+ %idx = mul i64 %iv.ext, %stride
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv.ext
+ store i64 %ld, ptr %gep.st
+ %iv.next = add i16 %iv, 1
+ %iv.next.ext = sext i16 %iv.next to i32
+ %ec = icmp sle i32 %iv.next.ext, %M
+ br i1 %ec, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @known_non_unit_via_range_attr(ptr noalias %out, ptr %p, i64 range(i64 4, 8) %stride) {
+; CHECK-LABEL: VPlan for loop in 'known_non_unit_via_range_attr'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%gep.out> = getelementptr inbounds ir<%out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%l>, ir<%gep.out>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+; CHECK-NEXT: IR %l = load i32, ptr %gep, align 4
+; CHECK-NEXT: IR %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+; CHECK-NEXT: IR store i32 %l, ptr %gep.out, align 4
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %ec = icmp eq i64 %iv.next, 1024
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare void @llvm.assume(i1 noundef)
+
+; TODO: We shouldn't speculate `%stride == 1` as it's known to be false via assume.
+define void @known_non_unit_via_assume(ptr noalias %out, ptr %p, i64 %stride) {
+; CHECK-LABEL: VPlan for loop in 'known_non_unit_via_assume'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %non_unit = icmp uge i64 %stride, 4
+; CHECK-NEXT: IR call void @llvm.assume(i1 %non_unit)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%gep.out> = getelementptr inbounds ir<%out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%l>, ir<%gep.out>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+; CHECK-NEXT: IR %l = load i32, ptr %gep, align 4
+; CHECK-NEXT: IR %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+; CHECK-NEXT: IR store i32 %l, ptr %gep.out, align 4
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %ec = icmp eq i64 %iv.next, 1024
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %non_unit = icmp uge i64 %stride, 4
+ call void @llvm.assume(i1 %non_unit)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The loaded stride has range metadata excluding unit stride.
+define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
+; CHECK-LABEL: VPlan for loop in 'known_non_unit_via_load_range'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %stride = load i64, ptr %stride.ptr, align 8, !range !0
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%p>, ir<%idx>
+; CHECK-NEXT: EMIT-SCALAR ir<%l> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%gep.out> = getelementptr inbounds ir<%out>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%l>, ir<%gep.out>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1024>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %idx = mul i64 %iv, %stride
+; CHECK-NEXT: IR %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+; CHECK-NEXT: IR %l = load i32, ptr %gep, align 4
+; CHECK-NEXT: IR %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+; CHECK-NEXT: IR store i32 %l, ptr %gep.out, align 4
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %ec = icmp eq i64 %iv.next, 1024
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+ ptr %stride.ptr) {
+entry:
+ %stride = load i64, ptr %stride.ptr, align 8, !range !0
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{i64 4, i64 8}
+
; Keep this in sync with the same under LoopVectorize/
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
index ca5e4c2ed8971..cc539644571ce 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll
@@ -41,22 +41,22 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -139,18 +139,18 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[HEADER]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @non_constant_btc(
@@ -159,8 +159,8 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF3:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
@@ -170,32 +170,32 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[TMP1]], <4 x i64> poison)
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP20]], <4 x i1> [[TMP1]])
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[TMP17]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP1]], <4 x i64> poison)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP5]], <4 x i1> [[TMP1]])
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP25:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF3]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[TMP25]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = mul i64 [[TMP25]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP25]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_CONTINUE4]], label %[[PRED_STORE_CONTINUE2]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @non_constant_btc(
@@ -205,21 +205,21 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) v
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH1]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP4]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
@@ -359,10 +359,10 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
;
; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[SCALAR_PH:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -370,7 +370,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -410,18 +410,18 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT1:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
@@ -433,8 +433,8 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT1]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT1]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_as_btc(
@@ -487,10 +487,10 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[SCALAR_PH:.*]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[SCALAR_PH1:.*]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -498,7 +498,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -575,9 +575,9 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -585,7 +585,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -594,55 +594,55 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = and i64 [[N_RND_UP]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP1]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[MIDDLE_BLOCK]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP2]], <4 x i64> poison)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP4]], <4 x i1> [[TMP2]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[MIDDLE_BLOCK]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[HEADER]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_dependent_btc(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_IF3:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> <i1 true, i1 true, i1 false, i1 false>)
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_IF7:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF1]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_IF7]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -650,8 +650,8 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_IF7]], label %[[HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
@@ -671,10 +671,10 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
@@ -683,15 +683,15 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -699,7 +699,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
@@ -707,9 +707,9 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -717,7 +717,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscal
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -754,22 +754,22 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -854,64 +854,64 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[HEADER]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_checks_order(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP2:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF3:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[SMAX]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[TMP1]], <4 x i64> poison)
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP20]], <4 x i1> [[TMP1]])
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[TMP17]], 4
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP1]], <4 x i64> poison)
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP5]], <4 x i1> [[TMP1]])
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP25:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF3]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[TMP25]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = mul i64 [[TMP25]], [[STRIDE]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP25]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_CONTINUE4]], label %[[PRED_STORE_CONTINUE2]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
@@ -922,21 +922,21 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH1]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP4]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT: store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
@@ -1036,29 +1036,29 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP3]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -1112,55 +1112,55 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = and i64 [[N_RND_UP]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP1]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[MIDDLE_BLOCK]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP2]], <4 x i64> poison)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_GATHER]], ptr align 8 [[TMP4]], <4 x i1> [[TMP2]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV-FOLD-TAIL: [[EXIT]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[MIDDLE_BLOCK]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-FOLD-TAIL: [[HEADER]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP17:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_IF3:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF3]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> splat (i1 true))
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE4]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_IF7:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_IF7]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF1]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_IF7]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1168,8 +1168,8 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_IF7]], label %[[HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
@@ -1189,10 +1189,10 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
@@ -1201,15 +1201,15 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1217,7 +1217,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
@@ -1225,9 +1225,9 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1235,7 +1235,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
@@ -1294,8 +1294,8 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
@@ -1311,10 +1311,10 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[HEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1322,7 +1322,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1350,8 +1350,8 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_MEMCHECK]]:
@@ -1359,17 +1359,17 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_STORE_CONTINUE6:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_STORE_CONTINUE6]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_BODY]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> splat (i1 true))
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER1]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1377,7 +1377,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL: [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: ret void
;
@@ -1450,36 +1450,36 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = and i64 [[TMP0]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
@@ -1555,8 +1555,8 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = and i64 [[N_RND_UP]], 3
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
@@ -1564,21 +1564,21 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP6]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP45:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP2]], <4 x i64> poison)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP5]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP46:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP30]], <4 x i1> [[TMP2]], <4 x i64> poison)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP4:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP4]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP45:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP3]], <4 x i64> poison)
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP46:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP5]], <4 x i1> [[TMP3]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP47:%.*]] = add <4 x i64> [[TMP45]], [[TMP46]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP54:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP47]], ptr align 8 [[TMP54]], <4 x i1> [[TMP2]])
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[TMP5]], 4
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP47]], ptr align 8 [[TMP7]], <4 x i1> [[TMP3]])
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP56:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP56]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-FOLD-TAIL-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL: [[SCALAR_PH]]:
@@ -1604,15 +1604,15 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE2:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE2]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP53:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[TMP53]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE4]]:
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br i1 [[DIFF_CHECK]], label %[[PRED_LOAD_IF3]], label %[[PRED_LOAD_IF5:.*]]
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_IF5]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[PRED_LOAD_CONTINUE6:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_CONTINUE6]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[TMP29:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> splat (i1 true), <4 x i64> poison)
@@ -1622,10 +1622,10 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV-FOLD_TAIL: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-FOLD_TAIL: [[PRED_LOAD_IF3]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL: [[HEADER]]:
-; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[PRED_LOAD_IF3]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1653,9 +1653,9 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-SCALABLE: [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 5
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = add nsw i64 [[TMP4]], -1
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP7:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-NO-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], -1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP6:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-SCALABLE-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE: [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
@@ -1665,12 +1665,12 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV-SCALABLE: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP10:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP8:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP8]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP11]], align 8
@@ -1680,15 +1680,15 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-NO-MV-SCALABLE: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE: [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-NO-MV-SCALABLE: [[HEADER]]:
-; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-NO-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-NO-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-NO-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1699,7 +1699,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-NO-MV-SCALABLE: [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT: ret void
;
@@ -1708,9 +1708,9 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-SCALABLE-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV-SCALABLE: [[HEADER]]:
-; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV-SCALABLE: [[HEADER1]]:
+; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1721,7 +1721,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-SCALABLE-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-SCALABLE-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE: [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index a1b1ac6207c13..4fa1b7cf31c68 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -18,20 +18,20 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic(
@@ -267,10 +267,10 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @byte_gep_scaled_stride(
@@ -293,10 +293,10 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -400,6 +400,94 @@ exit:
ret void
}
+; Like @byte_gep_under_scaled_stride, but with an extra-narrow use of %stride
+; in the loop to exercise MV replacement through `trunc ... to i1`.
+define void @byte_gep_under_scaled_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @byte_gep_under_scaled_stride_extra_narrow_use(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDE_I1:%.*]] = trunc i64 [[STRIDE]] to i1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 2)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P]], <4 x i64> [[TMP19]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = shl <4 x i64> [[WIDE_LOAD]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = select i1 [[STRIDE_I1]], <4 x i64> [[WIDE_LOAD]], <4 x i64> [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP5]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @byte_gep_under_scaled_stride_extra_narrow_use(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[STRIDE_I1:%.*]] = trunc i64 [[STRIDE]] to i1
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 2)
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P]], <4 x i64> [[TMP19]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = shl <4 x i64> [[WIDE_LOAD]], splat (i64 1)
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = select i1 [[STRIDE_I1]], <4 x i64> [[WIDE_LOAD]], <4 x i64> [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %stride.i1 = trunc i64 %stride to i1
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %stride.x8 = mul i64 %stride, 4
+ %idx = mul i64 %iv, %stride.x8
+
+ %gep.ld = getelementptr i8, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %mul = mul i64 %ld, 2
+ %val = select i1 %stride.i1, i64 %ld, i64 %mul
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
; Another variation for constant multiplier with byte gep. This time the
; multiplier is bigger than load access type so this cannot be speculated for
; unit-strideness.
@@ -424,7 +512,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -450,7 +538,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -502,7 +590,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -528,7 +616,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -578,7 +666,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -590,26 +678,26 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P]], <4 x i64> [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV: [[SCALAR_PH1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
@@ -617,8 +705,8 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -667,7 +755,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -693,7 +781,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -733,7 +821,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P0]], <4 x i64> [[TMP0]]
@@ -741,15 +829,15 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @shared_stride(
@@ -772,7 +860,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -789,7 +877,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -823,57 +911,57 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; COMPARE-NO-MV-LABEL: define void @independent_strides(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE0]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE1]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P0]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP3:%.*]] = getelementptr i64, ptr [[P1]], <4 x i64> [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = add <4 x i64> [[TMP25]], [[TMP33]]
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP34]], ptr [[TMP35]], align 8
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD2:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @independent_strides(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE0]], 1
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE1]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP0]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
+; COMPARE-LAA-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -891,7 +979,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -930,27 +1018,27 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-NO-MV: [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P0]], <4 x i64> [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i64, ptr [[P1]], <4 x i64> [[TMP6]]
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i64, ptr [[P0]], <4 x i64> [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P1]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @dependent_strides(
@@ -976,7 +1064,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -995,7 +1083,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1040,24 +1128,24 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P0]], <4 x i64> [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i64, ptr [[P1]], <4 x i64> [[TMP6]]
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P0]], <4 x i64> [[TMP6]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i64, ptr [[P1]], <4 x i64> [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP23]], [[WIDE_LOAD]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP10]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @dependent_strides_reverse_order(
@@ -1083,7 +1171,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1102,7 +1190,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1159,7 +1247,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1171,30 +1259,30 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P0]], <4 x i64> [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P0]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-LAA-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i8, ptr [[P1]], <4 x i64> [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i8, ptr [[P1]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i32> poison)
; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = sext <4 x i32> [[TMP27]] to <4 x i64>
; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP15]], [[TMP28]]
-; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV: [[SCALAR_PH1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
@@ -1206,8 +1294,8 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1263,7 +1351,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1275,30 +1363,30 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P1]], <4 x i64> [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P1]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = sext <4 x i32> [[TMP15]] to <4 x i64>
-; COMPARE-LAA-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i8, ptr [[P0]], <4 x i64> [[VEC_IND]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i8, ptr [[P0]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP28]], [[TMP16]]
-; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV: [[SCALAR_PH1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
@@ -1310,8 +1398,8 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1355,7 +1443,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
@@ -1363,15 +1451,15 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @strided_interleave(
@@ -1394,7 +1482,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1411,7 +1499,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1455,20 +1543,20 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[TMP0]], <4 x i64> [[TMP1]]
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @in_loop_base(
@@ -1489,7 +1577,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1505,7 +1593,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1558,10 +1646,10 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @base_not_in_ir(
@@ -1582,7 +1670,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1597,7 +1685,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1648,7 +1736,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1675,7 +1763,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -1729,7 +1817,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1757,7 +1845,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -1800,25 +1888,25 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1834,7 +1922,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -1859,7 +1947,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1875,7 +1963,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1927,7 +2015,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1943,16 +2031,16 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[SCALAR_PH:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -1960,7 +2048,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2013,7 +2101,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2029,7 +2117,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2037,9 +2125,9 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2047,7 +2135,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2084,25 +2172,25 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2118,7 +2206,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2144,7 +2232,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2160,7 +2248,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2193,29 +2281,29 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP3]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -2229,7 +2317,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2260,7 +2348,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2319,8 +2407,8 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
@@ -2336,10 +2424,10 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[HEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2347,7 +2435,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2382,36 +2470,36 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = and i64 [[TMP0]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP3]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
@@ -2428,7 +2516,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2470,7 +2558,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2524,7 +2612,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2550,7 +2638,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2592,20 +2680,20 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @nd_array_last_idx(
@@ -2625,7 +2713,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2639,7 +2727,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2688,7 +2776,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2700,26 +2788,26 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, <4 x i64> [[VEC_IND]], i64 42
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, <4 x i64> [[TMP0]], i64 42
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV: [[SCALAR_PH1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[IDX]], i64 42
@@ -2727,8 +2815,8 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP44:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -2774,7 +2862,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2799,7 +2887,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2839,20 +2927,20 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sext <4 x i32> [[BROADCAST_SPLAT]] to <4 x i64>
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP1]]
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @sext_stride(
@@ -2872,7 +2960,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2887,7 +2975,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP48:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2923,20 +3011,20 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_I64]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i32>
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i32>
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = mul <4 x i32> [[VEC_IND]], [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[P]], <4 x i32> [[TMP10]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i32> [[VEC_IND]], [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[P]], <4 x i32> [[TMP1]]
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP26]], ptr [[TMP27]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
@@ -2948,24 +3036,24 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX1]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV: [[SCALAR_PH1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
@@ -2974,8 +3062,8 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP49:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -3001,6 +3089,110 @@ exit:
ret void
}
+; Like the VPlan test of the same name, this extra-narrow use does not trigger
+; stride MV. Keep it here to cover the non-MV IR shape for `trunc ... to i1`.
+define void @trunc_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
+; COMPARE-NO-MV-LABEL: define void @trunc_stride_extra_narrow_use(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDE_I1:%.*]] = trunc i64 [[STRIDE_I64]] to i1
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_I64]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i32>
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = shl <4 x i32> [[TMP10]], splat (i32 2)
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = mul <4 x i32> [[VEC_IND]], [[TMP11]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P]], <4 x i32> [[TMP13]]
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = shl <4 x i32> [[TMP29]], splat (i32 1)
+; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = select i1 [[STRIDE_I1]], <4 x i32> [[TMP29]], <4 x i32> [[TMP30]]
+; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP31]], ptr [[TMP32]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP33]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @trunc_stride_extra_narrow_use(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[STRIDE_I1:%.*]] = trunc i64 [[STRIDE_I64]] to i1
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl i32 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = shl <4 x i32> [[TMP29]], splat (i32 1)
+; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = select i1 [[STRIDE_I1]], <4 x i32> [[TMP29]], <4 x i32> [[TMP30]]
+; COMPARE-LAA-MV-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i32> [[TMP31]], ptr [[TMP32]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP33]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
+; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
+; COMPARE-LAA-MV-NEXT: [[STRIDE_BYTE:%.*]] = mul i32 [[STRIDE]], 4
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE_BYTE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i32 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i32, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = mul i32 [[LD]], 2
+; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = select i1 [[STRIDE_I1]], i32 [[LD]], i32 [[MUL]]
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP52:![0-9]+]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %stride.i1 = trunc i64 %stride.i64 to i1
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %header ]
+ %stride = trunc i64 %stride.i64 to i32
+ %stride.byte = mul i32 %stride, 4
+ %iv.next = add nsw i32 %iv, 1
+
+ %idx = mul i32 %iv, %stride.byte
+
+ %gep.ld = getelementptr i8, ptr %p, i32 %idx
+ %ld = load i32, ptr %gep.ld, align 8
+
+ %mul = mul i32 %ld, 2
+ %val = select i1 %stride.i1, i32 %ld, i32 %mul
+
+ %gep.st = getelementptr i32, ptr %p.out, i32 %iv
+ store i32 %val, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i32 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
; %stride is usedc through both `trunc`/`sext` for different accesses.
define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride) {
; COMPARE-NO-MV-LABEL: define void @trunc_ext_stride(
@@ -3018,12 +3210,12 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND3:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = sext <4 x i32> [[VEC_IND]] to <4 x i64>
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul <4 x i16> [[VEC_IND3]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = mul <4 x i64> [[TMP8]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[P0]], <4 x i16> [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP4:%.*]] = getelementptr i32, ptr [[P0]], <4 x i64> [[TMP14]]
+; COMPARE-NO-MV-NEXT: [[VEC_IND3:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT6:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sext <4 x i32> [[VEC_IND]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i16> [[VEC_IND3]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[TMP0]], [[BROADCAST_SPLAT2]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[P0]], <4 x i16> [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP4:%.*]] = getelementptr i32, ptr [[P0]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; COMPARE-NO-MV-NEXT: [[TMP42:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP4]], <4 x i1> splat (i1 true), <4 x i32> poison)
; COMPARE-NO-MV-NEXT: [[TMP43:%.*]] = add <4 x i32> [[TMP34]], [[TMP42]]
@@ -3031,9 +3223,9 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP43]], ptr [[TMP44]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT4]] = add <4 x i16> [[VEC_IND3]], splat (i16 4)
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT6]] = add <4 x i16> [[VEC_IND3]], splat (i16 4)
; COMPARE-NO-MV-NEXT: [[TMP45:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
@@ -3047,11 +3239,11 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = mul i16 [[TMP0]], [[STRIDE_TRUNC]]
@@ -3064,7 +3256,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3084,7 +3276,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP51:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3121,6 +3313,86 @@ exit:
ret void
}
+; Stride is used through a chain of casts defined outside VPlan.
+define void @trunc_ext_chained_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @trunc_ext_chained_stride(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i64 [[STRIDE]] to i1
+; COMPARE-NO-MV-NEXT: [[STRIDE_SEXT:%.*]] = sext i1 [[STRIDE_TRUNC]] to i64
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_SEXT]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[REVERSE:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[REVERSE]], ptr [[TMP4]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @trunc_ext_chained_stride(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i64 [[STRIDE]] to i1
+; COMPARE-LAA-MV-NEXT: [[STRIDE_SEXT:%.*]] = sext i1 [[STRIDE_TRUNC]] to i64
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_SEXT]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-LAA-MV-NEXT: [[REVERSE:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[REVERSE]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %stride.trunc = trunc i64 %stride to i1
+ %stride.sext = sext i1 %stride.trunc to i64
+ br label %header
+
+header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.next = add nsw i64 %iv, 1
+
+ %idx = mul i64 %iv, %stride.sext
+
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld, align 8
+
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv
+ store i64 %ld, ptr %gep.st, align 8
+
+ %exitcond = icmp slt i64 %iv.next, 128
+ br i1 %exitcond, label %header, label %exit
+
+exit:
+ ret void
+}
+
; Check that we don't speculate unit-strided masked memory access if masked wide
; memory operation isn't legal (or that we properly pass the mask if it is).
define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
@@ -3146,10 +3418,10 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP24]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-NO-MV: [[IF]]:
+; COMPARE-NO-MV-NEXT: br label %[[LATCH:.*]]
+; COMPARE-NO-MV: [[LATCH]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic_masked(
@@ -3157,44 +3429,44 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF3:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = icmp sge <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
-; COMPARE-LAA-MV-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[TMP0]], <4 x i64> poison)
-; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
-; COMPARE-LAA-MV-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP20]], <4 x i1> [[TMP0]])
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP17]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP0]], <4 x i64> poison)
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP5]], <4 x i1> [[TMP0]])
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[C:%.*]] = icmp sge i64 [[IV]], [[X]]
-; COMPARE-LAA-MV-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[LATCH]]
-; COMPARE-LAA-MV: [[IF]]:
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[LATCH]]
-; COMPARE-LAA-MV: [[LATCH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF1]]:
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE2:.*]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF3]]:
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE4:.*]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE4]]:
+; COMPARE-LAA-MV-NEXT: [[TMP23:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF3]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[TMP23]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = icmp sge i64 [[TMP23]], [[X]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV: [[PRED_STORE_IF5]]:
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = mul i64 [[TMP23]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
+; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP23]]
+; COMPARE-LAA-MV-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
+; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE6]]:
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP53:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_CONTINUE4]], label %[[PRED_STORE_CONTINUE2]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE2]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -3243,7 +3515,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3266,7 +3538,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -3315,10 +3587,10 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic_strided_store(
@@ -3338,7 +3610,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3352,7 +3624,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3402,10 +3674,10 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @ptr_vec_use(
@@ -3430,7 +3702,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3446,7 +3718,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3484,24 +3756,24 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = mul <4 x i64> [[TMP16]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP17]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[WIDE_MASKED_GATHER]], [[TMP17]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP16]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_idx_vec_use(
@@ -3524,7 +3796,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3539,7 +3811,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP64:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3590,10 +3862,10 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
-; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @offset_stride_idx_vec_use(
@@ -3618,7 +3890,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP65:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3634,7 +3906,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP66:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3736,10 +4008,10 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[START_EXT]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[START_EXT]], i64 0
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = mul <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
@@ -3748,20 +4020,20 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr float, ptr [[DST]], <4 x i64> [[VEC_IND]]
; COMPARE-NO-MV-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> zeroinitializer, <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT: br label %[[LOOP:.*]]
; COMPARE-NO-MV: [[LOOP]]:
; COMPARE-NO-MV-NEXT: [[IV_0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_0_NEXT:%.*]], %[[LOOP]] ]
@@ -3771,7 +4043,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP51:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3791,7 +4063,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 4
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP67:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3806,7 +4078,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-LAA-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP64:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP68:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3828,4 +4100,358 @@ exit:
ret void
}
+; Used to crash because `getSymbolicMaxBackedgeTakenCount` is only available in
+; PSE but not SE.
+define void @stride_mv_predicated_btc(ptr noalias %p.out, ptr %p, i32 %M, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_mv_predicated_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[M:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add nuw i32 [[TMP0]], 1
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = trunc i32 [[SMAX]] to i16
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = add i16 1, [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp slt i16 [[TMP4]], 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[SMAX]], 65535
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = and i32 [[TMP1]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = trunc i32 [[N_VEC]] to i16
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[VEC_IND]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[TMP10]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP12]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP11]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP13]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[I:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[I_EXT:%.*]] = sext i16 [[I]] to i64
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[I_EXT]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 4
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[I_EXT]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 4
+; COMPARE-NO-MV-NEXT: [[I_NEXT]] = add i16 [[I]], 1
+; COMPARE-NO-MV-NEXT: [[I_NEXT_EXT:%.*]] = sext i16 [[I_NEXT]] to i32
+; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp sle i32 [[I_NEXT_EXT]], [[M]]
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP53:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_mv_predicated_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[M:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add nuw i32 [[TMP0]], 1
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = trunc i32 [[SMAX]] to i16
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = add i16 1, [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp slt i16 [[TMP4]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[SMAX]], 65535
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = and i32 [[TMP1]], 3
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP8]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = trunc i32 [[N_VEC]] to i16
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[VEC_IND]] to <4 x i64>
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[TMP10]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP11]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP13]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP69:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[I:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[I_EXT:%.*]] = sext i16 [[I]] to i64
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[I_EXT]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 4
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[I_EXT]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 4
+; COMPARE-LAA-MV-NEXT: [[I_NEXT]] = add i16 [[I]], 1
+; COMPARE-LAA-MV-NEXT: [[I_NEXT_EXT:%.*]] = sext i16 [[I_NEXT]] to i32
+; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp sle i32 [[I_NEXT_EXT]], [[M]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP70:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i16 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.ext = sext i16 %iv to i64
+ %idx = mul i64 %iv.ext, %stride
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv.ext
+ store i64 %ld, ptr %gep.st
+ %iv.next = add i16 %iv, 1
+ %iv.next.ext = sext i16 %iv.next to i32
+ %ec = icmp sle i32 %iv.next.ext, %M
+ br i1 %ec, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @known_non_unit_via_range_attr(ptr noalias %out, ptr %p, i64 range(i64 4, 8) %stride) {
+; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_range_attr(
+; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 range(i64 4, 8) [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP1]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-NO-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_range_attr(
+; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 range(i64 4, 8) [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[VECTOR_PH:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare void @llvm.assume(i1 noundef)
+
+; TODO: We shouldn't speculate `%stride == 1` as it's known to be false via assume.
+define void @known_non_unit_via_assume(ptr noalias %out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_assume(
+; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[NON_UNIT:%.*]] = icmp uge i64 [[STRIDE]], 4
+; COMPARE-NO-MV-NEXT: call void @llvm.assume(i1 [[NON_UNIT]])
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-NO-MV-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[LOOP:.*]]
+; COMPARE-NO-MV: [[LOOP]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_assume(
+; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[NON_UNIT:%.*]] = icmp uge i64 [[STRIDE]], 4
+; COMPARE-LAA-MV-NEXT: call void @llvm.assume(i1 [[NON_UNIT]])
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP0]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP71:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[LOOP:.*]]
+; COMPARE-LAA-MV: [[LOOP]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; COMPARE-LAA-MV-NEXT: [[GEP_OUT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[GEP_OUT]], align 4
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP72:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %non_unit = icmp uge i64 %stride, 4
+ call void @llvm.assume(i1 %non_unit)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The loaded stride has range metadata excluding unit stride.
+define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
+; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_load_range(
+; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], ptr [[STRIDE_PTR:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = load i64, ptr [[STRIDE_PTR]], align 8, !range [[RNG56:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP1]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-NO-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_load_range(
+; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], ptr [[STRIDE_PTR:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[VECTOR_PH:.*]]:
+; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = load i64, ptr [[STRIDE_PTR]], align 8, !range [[RNG73:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+ ptr %stride.ptr) {
+entry:
+ %stride = load i64, ptr %stride.ptr, align 8, !range !0
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{i64 4, i64 8}
+
; Keep this in sync with the same under VPlan/
>From e5bfadf944de737a2cfe659dde6695aafc6a8595 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 08:04:52 -0700
Subject: [PATCH 2/2] Fix formatting
---
.../Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll | 3 +--
llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index 45687811d808a..96b8f13386e53 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -3592,7 +3592,7 @@ exit:
}
; The loaded stride has range metadata excluding unit stride.
-define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
+define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p, ptr %stride.ptr) {
; CHECK-LABEL: VPlan for loop in 'known_non_unit_via_load_range'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
@@ -3650,7 +3650,6 @@ define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
- ptr %stride.ptr) {
entry:
%stride = load i64, ptr %stride.ptr, align 8, !range !0
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 4fa1b7cf31c68..2a14e3e959ce0 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -4387,7 +4387,7 @@ exit:
}
; The loaded stride has range metadata excluding unit stride.
-define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
+define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p, ptr %stride.ptr) {
; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_load_range(
; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], ptr [[STRIDE_PTR:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
@@ -4432,7 +4432,6 @@ define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
- ptr %stride.ptr) {
entry:
%stride = load i64, ptr %stride.ptr, align 8, !range !0
br label %loop
More information about the llvm-commits
mailing list