[llvm] [VPlan] Replace WideCanIV with CanIV when possible (PR #191276)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 9 12:20:07 PDT 2026
https://github.com/artagnon created https://github.com/llvm/llvm-project/pull/191276
Improve removeRedundantCanonicalIVs to replace WideCanonicalIV with CanonicalIV when possible. Unfortunately, the test mve-reg-pressure-spills could not be saved, and had to be removed.
>From 821fe57f2e2583ec3bee8870280e52c4ff3988fd Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 9 Apr 2026 20:04:44 +0100
Subject: [PATCH] [VPlan] Replace WideCanIV with CanIV when possible
Improve removeRedundantCanonicalIVs to replace WideCanonicalIV with
CanonicalIV when possible. Unfortunately, the test
mve-reg-pressure-spills could not be saved, and had to be removed.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 16 +-
.../sve-interleaved-masked-accesses.ll | 8 +-
.../LoopVectorize/ARM/active-lane-mask.ll | 14 +-
.../ARM/mve-gather-scatter-tailpred.ll | 4 +-
.../ARM/mve-reg-pressure-spills.ll | 268 ------------------
.../ARM/tail-folding-counting-down.ll | 3 +-
.../ARM/tail-folding-prefer-flag.ll | 33 +--
.../RISCV/interleaved-masked-access.ll | 16 +-
llvm/test/Transforms/LoopVectorize/as_cast.ll | 1 +
.../LoopVectorize/cast-induction.ll | 17 +-
.../first-order-recurrence-tail-folding.ll | 51 ++--
.../LoopVectorize/first-order-recurrence.ll | 27 +-
.../pr45679-fold-tail-by-masking.ll | 30 +-
.../strict-fadd-interleave-only.ll | 45 ++-
.../tail-folding-vectorization-factor-1.ll | 42 ++-
15 files changed, 128 insertions(+), 447 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 098d9975beabb..7fc4f4b5222f4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -679,15 +679,17 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
static void removeRedundantCanonicalIVs(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPCanonicalIVPHIRecipe *CanonicalIV = LoopRegion->getCanonicalIV();
- VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
- for (VPUser *U : CanonicalIV->users()) {
- WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
- if (WidenNewIV)
- break;
- }
+ auto *FoundWideIV =
+ find_if(CanonicalIV->users(), IsaPred<VPWidenCanonicalIVRecipe>);
+ if (FoundWideIV == CanonicalIV->user_end())
+ return;
+ auto *WidenNewIV = cast<VPWidenCanonicalIVRecipe>(*FoundWideIV);
- if (!WidenNewIV)
+ if (Plan.hasScalarVFOnly() || vputils::onlyScalarValuesUsed(WidenNewIV)) {
+ WidenNewIV->replaceAllUsesWith(CanonicalIV);
+ WidenNewIV->eraseFromParent();
return;
+ }
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index f352fa12e1093..44d17756f6aba 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -107,7 +107,7 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP14]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
; PREDICATED_TAIL_FOLDING: middle.block:
; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]]
@@ -233,7 +233,7 @@ define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP3:![0-9]+]]
; PREDICATED_TAIL_FOLDING: middle.block:
; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]]
@@ -363,7 +363,7 @@ define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias noca
; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP13]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
; PREDICATED_TAIL_FOLDING: middle.block:
; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]]
@@ -519,7 +519,7 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024)
; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP18]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
; PREDICATED_TAIL_FOLDING: middle.block:
; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
index 3ffafa404269b..df5afc66ed877 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/active-lane-mask.ll
@@ -17,21 +17,19 @@ define void @f0(ptr noalias %dst, ptr readonly %src, i64 %n) #0 {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX]], i64 [[N]])
+; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[TMP0]], i64 [[N]])
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP1]], <16 x i1> [[ACTIVE_LANE_MASK1]], <16 x i8> poison)
; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[ACTIVE_LANE_MASK1]], <16 x i8> poison)
; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 3)
; CHECK-NEXT: [[TMP5:%.*]] = mul <16 x i8> [[WIDE_MASKED_LOAD2]], splat (i8 3)
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 16
-; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP4]], ptr align 1 [[TMP6]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP4]], ptr align 1 [[TMP6]], <16 x i1> [[ACTIVE_LANE_MASK1]])
; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP5]], ptr align 1 [[TMP8]], <16 x i1> [[ACTIVE_LANE_MASK1]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[TMP0]], 32
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
index 3ed25a18d91f7..f1345281704b5 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
@@ -193,7 +193,7 @@ define void @test_stride3_4i32(ptr readonly %data, ptr noalias nocapture %dst, i
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: middle.block:
@@ -240,7 +240,7 @@ define void @test_stride4_4i32(ptr readonly %data, ptr noalias nocapture %dst, i
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
deleted file mode 100644
index 193bd60a9dfe2..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-spills.ll
+++ /dev/null
@@ -1,268 +0,0 @@
-; REQUIRES: asserts
-; RUN: opt -mcpu=cortex-m55 -passes=loop-vectorize -disable-output -debug-only=loop-vectorize,vplan -vplan-verify-each -vectorizer-consider-reg-pressure=false %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-NOPRESSURE
-; RUN: opt -mcpu=cortex-m55 -passes=loop-vectorize -disable-output -debug-only=loop-vectorize,vplan -vplan-verify-each -vectorizer-consider-reg-pressure=true %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PRESSURE
-; RUN: opt -mcpu=cortex-m55 -passes=loop-vectorize -disable-output -debug-only=loop-vectorize,vplan -vplan-verify-each %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PRESSURE
-
-target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
-target triple = "thumbv8.1m.main-unknown-none-eabihf"
-
-; In this function the spills make it not profitable to vectorize if considering
-; register pressure.
-define void @spills_not_profitable(ptr %in1, ptr %in2, ptr %out, i32 %n) {
-; CHECK-LABEL: LV: Checking a loop in 'spills_not_profitable'
-; CHECK: LV: Scalar loop costs: 86
-; CHECK-NOPRESSURE: Cost for VF 2: 394 (Estimated cost per lane: 197.0)
-; CHECK-NOPRESSURE: Cost for VF 4: 338 (Estimated cost per lane: 84.5)
-; CHECK-NOPRESSURE: LV: Selecting VF: 4
-; CHECK-PRESSURE: LV(REG): Cost of 50 from 25 spills of Generic::VectorRC
-; CHECK-PRESSURE-NEXT: Cost for VF 2: 444 (Estimated cost per lane: 222.0)
-; CHECK-PRESSURE: LV(REG): Cost of 50 from 25 spills of Generic::VectorRC
-; CHECK-PRESSURE-NEXT: Cost for VF 4: 388 (Estimated cost per lane: 97.0)
-; CHECK-PRESSURE: LV: Selecting VF: 1
-entry:
- %cmp = icmp eq i32 %n, 0
- br i1 %cmp, label %exit, label %for.body
-
-for.body:
- %i = phi i32 [ %inc, %for.body ], [ 0, %entry ]
- %x4 = phi float [ %x4.next, %for.body ], [ 0.000000e+00, %entry ]
- %x3 = phi float [ %x3.next, %for.body ], [ 0.000000e+00, %entry ]
- %x2 = phi float [ %x2.next, %for.body ], [ 0.000000e+00, %entry ]
- %x1 = phi float [ %x1.next, %for.body ], [ 0.000000e+00, %entry ]
- %x0 = phi float [ %x0.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc7 = phi float [ %acc7.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc6 = phi float [ %acc6.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc5 = phi float [ %acc5.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc4 = phi float [ %acc4.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc3 = phi float [ %acc3.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc2 = phi float [ %acc2.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc1 = phi float [ %acc1.next, %for.body ], [ 0.000000e+00, %entry ]
- %acc0 = phi float [ %acc0.next, %for.body ], [ 0.000000e+00, %entry ]
- %in1.addr = phi ptr [ %in1.addr.next, %for.body ], [ %in1, %entry ]
- %in2.addr = phi ptr [ %in2.addr.next, %for.body ], [ %in2, %entry ]
- %incdec.ptr = getelementptr inbounds nuw i8, ptr %in1.addr, i32 4
- %0 = load float, ptr %in1.addr, align 4
- %incdec.ptr1 = getelementptr inbounds nuw i8, ptr %in2.addr, i32 4
- %1 = load float, ptr %in2.addr, align 4
- %mul = fmul fast float %0, %x0
- %add = fadd fast float %mul, %acc0
- %mul2 = fmul fast float %0, %x1
- %add3 = fadd fast float %mul2, %acc1
- %mul4 = fmul fast float %0, %x2
- %add5 = fadd fast float %mul4, %acc2
- %mul6 = fmul fast float %0, %x3
- %add7 = fadd fast float %mul6, %acc3
- %mul8 = fmul fast float %0, %x4
- %add9 = fadd fast float %mul8, %acc4
- %mul10 = fmul fast float %1, %0
- %add11 = fadd fast float %mul10, %acc7
- %incdec.ptr12 = getelementptr inbounds nuw i8, ptr %in1.addr, i32 8
- %2 = load float, ptr %incdec.ptr, align 4
- %incdec.ptr13 = getelementptr inbounds nuw i8, ptr %in2.addr, i32 8
- %x0.next = load float, ptr %incdec.ptr1, align 4
- %mul14 = fmul fast float %2, %x1
- %add15 = fadd fast float %add, %mul14
- %mul16 = fmul fast float %2, %x2
- %add17 = fadd fast float %add3, %mul16
- %mul18 = fmul fast float %2, %x3
- %add19 = fadd fast float %add5, %mul18
- %mul20 = fmul fast float %2, %x4
- %add21 = fadd fast float %add7, %mul20
- %mul22 = fmul fast float %2, %1
- %add23 = fadd fast float %mul22, %acc6
- %mul24 = fmul fast float %x0.next, %2
- %add25 = fadd fast float %add11, %mul24
- %incdec.ptr26 = getelementptr inbounds nuw i8, ptr %in1.addr, i32 12
- %4 = load float, ptr %incdec.ptr12, align 4
- %incdec.ptr27 = getelementptr inbounds nuw i8, ptr %in2.addr, i32 12
- %x1.next = load float, ptr %incdec.ptr13, align 4
- %mul28 = fmul fast float %4, %x2
- %add29 = fadd fast float %add15, %mul28
- %mul30 = fmul fast float %4, %x3
- %add31 = fadd fast float %add17, %mul30
- %mul32 = fmul fast float %4, %x4
- %add33 = fadd fast float %add19, %mul32
- %mul34 = fmul fast float %4, %1
- %add35 = fadd fast float %mul34, %acc5
- %mul36 = fmul fast float %4, %x0.next
- %add37 = fadd fast float %add23, %mul36
- %mul38 = fmul fast float %x1.next, %4
- %add39 = fadd fast float %add25, %mul38
- %incdec.ptr40 = getelementptr inbounds nuw i8, ptr %in1.addr, i32 16
- %6 = load float, ptr %incdec.ptr26, align 4
- %incdec.ptr41 = getelementptr inbounds nuw i8, ptr %in2.addr, i32 16
- %x2.next = load float, ptr %incdec.ptr27, align 4
- %mul42 = fmul fast float %6, %x3
- %add43 = fadd fast float %add29, %mul42
- %mul44 = fmul fast float %6, %x4
- %acc1.next = fadd fast float %add31, %mul44
- %mul46 = fmul fast float %6, %1
- %add47 = fadd fast float %add9, %mul46
- %mul48 = fmul fast float %6, %x0.next
- %add49 = fadd fast float %add35, %mul48
- %mul50 = fmul fast float %6, %x1.next
- %add51 = fadd fast float %add37, %mul50
- %mul52 = fmul fast float %x2.next, %6
- %add53 = fadd fast float %add39, %mul52
- %incdec.ptr54 = getelementptr inbounds nuw i8, ptr %in1.addr, i32 20
- %8 = load float, ptr %incdec.ptr40, align 4
- %incdec.ptr55 = getelementptr inbounds nuw i8, ptr %in2.addr, i32 20
- %x3.next = load float, ptr %incdec.ptr41, align 4
- %mul56 = fmul fast float %8, %x4
- %acc0.next = fadd fast float %add43, %mul56
- %mul58 = fmul fast float %8, %1
- %add59 = fadd fast float %add21, %mul58
- %mul60 = fmul fast float %8, %x0.next
- %add61 = fadd fast float %add47, %mul60
- %mul62 = fmul fast float %8, %x1.next
- %add63 = fadd fast float %add49, %mul62
- %mul64 = fmul fast float %8, %x2.next
- %add65 = fadd fast float %add51, %mul64
- %mul66 = fmul fast float %x3.next, %8
- %add67 = fadd fast float %add53, %mul66
- %in1.addr.next = getelementptr inbounds nuw i8, ptr %in1.addr, i32 24
- %10 = load float, ptr %incdec.ptr54, align 4
- %in2.addr.next = getelementptr inbounds nuw i8, ptr %in2.addr, i32 24
- %x4.next = load float, ptr %incdec.ptr55, align 4
- %mul70 = fmul fast float %10, %1
- %acc2.next = fadd fast float %add33, %mul70
- %mul72 = fmul fast float %10, %x0.next
- %acc3.next = fadd fast float %add59, %mul72
- %mul74 = fmul fast float %10, %x1.next
- %acc4.next = fadd fast float %add61, %mul74
- %mul76 = fmul fast float %10, %x2.next
- %acc5.next = fadd fast float %add63, %mul76
- %mul78 = fmul fast float %10, %x3.next
- %acc6.next = fadd fast float %add65, %mul78
- %mul80 = fmul fast float %x4.next, %10
- %acc7.next = fadd fast float %add67, %mul80
- %inc = add nuw i32 %i, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %exit, label %for.body
-
-exit:
- %acc0.exit = phi float [ 0.000000e+00, %entry ], [ %acc0.next, %for.body ]
- %acc1.exit = phi float [ 0.000000e+00, %entry ], [ %acc1.next, %for.body ]
- %acc2.exit = phi float [ 0.000000e+00, %entry ], [ %acc2.next, %for.body ]
- %acc3.exit = phi float [ 0.000000e+00, %entry ], [ %acc3.next, %for.body ]
- %acc4.exit = phi float [ 0.000000e+00, %entry ], [ %acc4.next, %for.body ]
- %acc5.exit = phi float [ 0.000000e+00, %entry ], [ %acc5.next, %for.body ]
- %acc6.exit = phi float [ 0.000000e+00, %entry ], [ %acc6.next, %for.body ]
- %acc7.exit = phi float [ 0.000000e+00, %entry ], [ %acc7.next, %for.body ]
- store float %acc0.exit, ptr %out, align 4
- %arrayidx82 = getelementptr inbounds nuw i8, ptr %out, i32 4
- store float %acc1.exit, ptr %arrayidx82, align 4
- %arrayidx83 = getelementptr inbounds nuw i8, ptr %out, i32 8
- store float %acc2.exit, ptr %arrayidx83, align 4
- %arrayidx84 = getelementptr inbounds nuw i8, ptr %out, i32 12
- store float %acc3.exit, ptr %arrayidx84, align 4
- %arrayidx85 = getelementptr inbounds nuw i8, ptr %out, i32 16
- store float %acc4.exit, ptr %arrayidx85, align 4
- %arrayidx86 = getelementptr inbounds nuw i8, ptr %out, i32 20
- store float %acc5.exit, ptr %arrayidx86, align 4
- %arrayidx87 = getelementptr inbounds nuw i8, ptr %out, i32 24
- store float %acc6.exit, ptr %arrayidx87, align 4
- %arrayidx88 = getelementptr inbounds nuw i8, ptr %out, i32 28
- store float %acc7.exit, ptr %arrayidx88, align 4
- ret void
-}
-
-; In this function we have spills but it is still profitable to vectorize when
-; considering register pressure.
-define void @spills_profitable(ptr %in1, ptr %in2, ptr %out, i32 %n, i32 %m) {
-; CHECK-LABEL: LV: Checking a loop in 'spills_profitable'
-; CHECK: LV: Scalar loop costs: 54
-; CHECK-NOPRESSURE: Cost for VF 2: 1530 (Estimated cost per lane: 765.0)
-; CHECK-NOPRESSURE: Cost for VF 4: 38 (Estimated cost per lane: 9.5)
-; CHECK-PRESSURE: LV(REG): Cost of 4 from 2 spills of Generic::ScalarRC
-; CHECK-PRESSURE-NEXT: Cost for VF 2: 1534 (Estimated cost per lane: 767.0)
-; CHECK-PRESSURE: LV(REG): Cost of 6 from 3 spills of Generic::VectorRC
-; CHECK-PRESSURE-NEXT: Cost for VF 4: 44 (Estimated cost per lane: 11.0)
-; CHECK: LV: Selecting VF: 4
-entry:
- %cmp = icmp eq i32 %n, 0
- br i1 %cmp, label %exit, label %for.body.preheader
-
-for.body.preheader:
- %add.ptr3.idx = mul i32 %m, 12
- %add.ptr3 = getelementptr inbounds nuw i8, ptr %in1, i32 %add.ptr3.idx
- %add.ptr1.idx = shl i32 %m, 3
- %add.ptr1 = getelementptr inbounds nuw i8, ptr %in1, i32 %add.ptr1.idx
- %add.ptr = getelementptr inbounds nuw i32, ptr %in1, i32 %m
- br label %for.body
-
-for.body:
- %i = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
- %acc3 = phi i64 [ %acc3.next, %for.body ], [ 0, %for.body.preheader ]
- %acc2 = phi i64 [ %acc2.next, %for.body ], [ 0, %for.body.preheader ]
- %acc1 = phi i64 [ %acc1.next, %for.body ], [ 0, %for.body.preheader ]
- %acc0 = phi i64 [ %acc0.next, %for.body ], [ 0, %for.body.preheader ]
- %in2.addr = phi ptr [ %in2.addr.next, %for.body ], [ %in2, %for.body.preheader ]
- %px3 = phi ptr [ %px3.next, %for.body ], [ %add.ptr3, %for.body.preheader ]
- %px2 = phi ptr [ %px2.next, %for.body ], [ %add.ptr1, %for.body.preheader ]
- %px1 = phi ptr [ %px1.next, %for.body ], [ %add.ptr, %for.body.preheader ]
- %px0 = phi ptr [ %px0.next, %for.body ], [ %in1, %for.body.preheader ]
- %incdec.ptr = getelementptr inbounds nuw i8, ptr %in2.addr, i32 4
- %0 = load i32, ptr %in2.addr, align 4
- %incdec.ptr4 = getelementptr inbounds nuw i8, ptr %px0, i32 4
- %1 = load i32, ptr %px0, align 4
- %incdec.ptr5 = getelementptr inbounds nuw i8, ptr %px1, i32 4
- %2 = load i32, ptr %px1, align 4
- %incdec.ptr6 = getelementptr inbounds nuw i8, ptr %px2, i32 4
- %3 = load i32, ptr %px2, align 4
- %incdec.ptr7 = getelementptr inbounds nuw i8, ptr %px3, i32 4
- %4 = load i32, ptr %px3, align 4
- %conv = sext i32 %1 to i64
- %conv8 = sext i32 %0 to i64
- %mul9 = mul nsw i64 %conv, %conv8
- %add = add nsw i64 %mul9, %acc0
- %conv10 = sext i32 %2 to i64
- %mul12 = mul nsw i64 %conv10, %conv8
- %add13 = add nsw i64 %mul12, %acc1
- %conv14 = sext i32 %3 to i64
- %mul16 = mul nsw i64 %conv14, %conv8
- %add17 = add nsw i64 %mul16, %acc2
- %conv18 = sext i32 %4 to i64
- %mul20 = mul nsw i64 %conv18, %conv8
- %add21 = add nsw i64 %mul20, %acc3
- %in2.addr.next = getelementptr inbounds nuw i8, ptr %in2.addr, i32 8
- %5 = load i32, ptr %incdec.ptr, align 4
- %px0.next = getelementptr inbounds nuw i8, ptr %px0, i32 8
- %6 = load i32, ptr %incdec.ptr4, align 4
- %px1.next = getelementptr inbounds nuw i8, ptr %px1, i32 8
- %7 = load i32, ptr %incdec.ptr5, align 4
- %px2.next = getelementptr inbounds nuw i8, ptr %px2, i32 8
- %8 = load i32, ptr %incdec.ptr6, align 4
- %px3.next = getelementptr inbounds nuw i8, ptr %px3, i32 8
- %9 = load i32, ptr %incdec.ptr7, align 4
- %conv27 = sext i32 %6 to i64
- %conv28 = sext i32 %5 to i64
- %mul29 = mul nsw i64 %conv27, %conv28
- %acc0.next = add nsw i64 %add, %mul29
- %conv31 = sext i32 %7 to i64
- %mul33 = mul nsw i64 %conv31, %conv28
- %acc1.next = add nsw i64 %add13, %mul33
- %conv35 = sext i32 %8 to i64
- %mul37 = mul nsw i64 %conv35, %conv28
- %acc2.next = add nsw i64 %add17, %mul37
- %conv39 = sext i32 %9 to i64
- %mul41 = mul nsw i64 %conv39, %conv28
- %acc3.next = add nsw i64 %add21, %mul41
- %inc = add nuw nsw i32 %i, 1
- %exitcond = icmp eq i32 %inc, %n
- br i1 %exitcond, label %exit, label %for.body
-
-exit:
- %acc0.exit = phi i64 [ 0, %entry ], [ %acc0.next, %for.body ]
- %acc1.exit = phi i64 [ 0, %entry ], [ %acc1.next, %for.body ]
- %acc2.exit = phi i64 [ 0, %entry ], [ %acc2.next, %for.body ]
- %acc3.exit = phi i64 [ 0, %entry ], [ %acc3.next, %for.body ]
- store i64 %acc0.exit, ptr %out, align 8
- %arrayidx43 = getelementptr inbounds nuw i8, ptr %out, i32 8
- store i64 %acc1.exit, ptr %arrayidx43, align 8
- %arrayidx44 = getelementptr inbounds nuw i8, ptr %out, i32 16
- store i64 %acc2.exit, ptr %arrayidx44, align 8
- %arrayidx45 = getelementptr inbounds nuw i8, ptr %out, i32 24
- store i64 %acc3.exit, ptr %arrayidx45, align 8
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index 68cf39e3406e5..8c4717f89bbcc 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -17,8 +17,7 @@ define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapt
; COMMON-LABEL: @sgt_loopguard(
; COMMON: vector.body:
-; CHECK-TF: %[[VIVELEM0:.*]] = extractelement <16 x i32> %vec.iv, i32 0
-; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[VIVELEM0]], i32 %N)
+; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %{{.*}}, i32 %N)
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
index 881a6e608dda1..6ad51a08c8ea5 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
@@ -51,27 +51,20 @@ for.body:
define void @interleave4(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C, i32 %N) #0 {
; PREDFLAG-LABEL: interleave4(
-; PREDFLAG: %[[ADD2:.*]] = add i32 %index, 4
-; PREDFLAG: %[[ADD3:.*]] = add i32 %index, 8
-; PREDFLAG: %[[ADD4:.*]] = add i32 %index, 12
-; PREDFLAG: %[[ALM1:active.lane.mask.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
-; PREDFLAG: %[[ALM2:active.lane.mask.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %[[ADD2]], i32 %N)
-; PREDFLAG: %[[ALM3:active.lane.mask.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %[[ADD3]], i32 %N)
-; PREDFLAG: %[[ALM4:active.lane.mask.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %[[ADD4]], i32 %N)
+; PREDFLAG: %[[ALM:active.lane.mask.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
+; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM]],{{.*}}
;
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM1]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM2]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM3]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM4]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM1]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM2]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM3]],{{.*}}
-; PREDFLAG: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %[[ALM4]],{{.*}}
-;
-; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM1]])
-; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM2]])
-; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM3]])
-; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM4]])
+; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM]])
+; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM]])
+; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM]])
+; PREDFLAG: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %[[ALM]])
;
entry:
%cmp8 = icmp sgt i32 %N, 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index b62d6da2f2e2d..c22ac8eafca10 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -91,7 +91,7 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_DATA-NEXT: [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
; PREDICATED_DATA-NEXT: call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP12]], <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
; PREDICATED_DATA-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; PREDICATED_DATA-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_DATA-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_DATA-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; PREDICATED_DATA-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; PREDICATED_DATA: middle.block:
@@ -110,14 +110,14 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
; PREDICATED_DATA-WITH-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; PREDICATED_DATA-WITH-EVL: vector.body:
-; PREDICATED_DATA-WITH-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_DATA-WITH-EVL-NEXT: [[CURRENT_ITERATION_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL:%.*]] = phi i32 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP2:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP3:%.*]] = shl i32 [[EVL_BASED_IV]], 1
+; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP3:%.*]] = shl i32 [[CURRENT_ITERATION_IV]], 1
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = shl nuw nsw i32 [[TMP1]], 1
@@ -134,7 +134,7 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_DATA-WITH-EVL-NEXT: [[INTERLEAVED_MASK4:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP2]])
; PREDICATED_DATA-WITH-EVL-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP11]])
; PREDICATED_DATA-WITH-EVL-NEXT: call void @llvm.vp.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP10]], <vscale x 32 x i1> [[INTERLEAVED_MASK4]], i32 [[INTERLEAVE_EVL3]])
-; PREDICATED_DATA-WITH-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
+; PREDICATED_DATA-WITH-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP1]], [[CURRENT_ITERATION_IV]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -273,7 +273,7 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_DATA-NEXT: [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
; PREDICATED_DATA-NEXT: call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP17]], <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
; PREDICATED_DATA-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; PREDICATED_DATA-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_DATA-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_DATA-NEXT: [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; PREDICATED_DATA-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; PREDICATED_DATA: middle.block:
@@ -292,14 +292,14 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
; PREDICATED_DATA-WITH-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
; PREDICATED_DATA-WITH-EVL: vector.body:
-; PREDICATED_DATA-WITH-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_DATA-WITH-EVL-NEXT: [[CURRENT_ITERATION_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL:%.*]] = phi i32 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP2:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP3:%.*]] = shl i32 [[EVL_BASED_IV]], 2
+; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP3:%.*]] = shl i32 [[CURRENT_ITERATION_IV]], 2
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = shl nuw nsw i32 [[TMP1]], 2
@@ -320,7 +320,7 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
; PREDICATED_DATA-WITH-EVL-NEXT: [[INTERLEAVED_MASK4:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP2]])
; PREDICATED_DATA-WITH-EVL-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]])
; PREDICATED_DATA-WITH-EVL-NEXT: call void @llvm.vp.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr align 1 [[TMP15]], <vscale x 64 x i1> [[INTERLEAVED_MASK4]], i32 [[INTERLEAVE_EVL3]])
-; PREDICATED_DATA-WITH-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
+; PREDICATED_DATA-WITH-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP1]], [[CURRENT_ITERATION_IV]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/as_cast.ll b/llvm/test/Transforms/LoopVectorize/as_cast.ll
index 2437bcfde08f9..3d0306827b001 100644
--- a/llvm/test/Transforms/LoopVectorize/as_cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/as_cast.ll
@@ -19,6 +19,7 @@ loop:
; CHECK: store i64 [[ID1]], ptr [[GEP1]]
; CHECK: pred.store.if1:
+; CHECK: add i64 %index, 1
; CHECK: [[ID2:%.*]] = add i64 %{{.*}}, 1
; CHECK: [[AS2:%.*]] = addrspacecast ptr addrspace(1) %in to ptr
; CHECK: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[AS2]], i64 [[ID2]]
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
index e5069d97f3117..bd28d2f235b53 100644
--- a/llvm/test/Transforms/LoopVectorize/cast-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -272,25 +272,24 @@ define void @cast_induction_tail_folding(ptr %A) {
; IC2: [[VECTOR_PH]]:
; IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; IC2: [[VECTOR_BODY]]:
-; IC2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; IC2-NEXT: [[INDEX1:%.*]] = add i32 [[INDEX]], 1
-; IC2-NEXT: [[TMP2:%.*]] = icmp ule i32 [[INDEX]], 2
+; IC2-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; IC2-NEXT: [[TMP3:%.*]] = icmp ule i32 [[INDEX1]], 2
-; IC2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; IC2-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; IC2: [[PRED_STORE_IF]]:
-; IC2-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64
+; IC2-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX1]] to i64
; IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
-; IC2-NEXT: store i32 [[INDEX]], ptr [[TMP5]], align 4
+; IC2-NEXT: store i32 [[INDEX1]], ptr [[TMP5]], align 4
; IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
; IC2: [[PRED_STORE_CONTINUE]]:
; IC2-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; IC2: [[PRED_STORE_IF1]]:
-; IC2-NEXT: [[TMP6:%.*]] = sext i32 [[INDEX1]] to i64
+; IC2-NEXT: [[TMP9:%.*]] = add i32 [[INDEX1]], 1
+; IC2-NEXT: [[TMP6:%.*]] = sext i32 [[TMP9]] to i64
; IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
-; IC2-NEXT: store i32 [[INDEX1]], ptr [[TMP7]], align 4
+; IC2-NEXT: store i32 [[TMP9]], ptr [[TMP7]], align 4
; IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; IC2: [[PRED_STORE_CONTINUE2]]:
-; IC2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; IC2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX1]], 2
; IC2-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
; IC2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; IC2: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
index 10e4eaf63cfd6..165ecdfd314ee 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
@@ -224,18 +224,16 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; VF1IC2-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 33, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; VF1IC2-NEXT: [[VEC_IV:%.*]] = add i64 [[TMP0]], 0
; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
-; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
-; VF1IC2-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV1]], [[TRIP_COUNT_MINUS_1]]
+; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; VF1IC2: [[PRED_LOAD_IF]]:
-; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[VEC_IV]]
+; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP0]]
; VF1IC2-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
; VF1IC2-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; VF1IC2: [[PRED_LOAD_CONTINUE]]:
; VF1IC2-NEXT: [[TMP5:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
-; VF1IC2-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
+; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
; VF1IC2: [[PRED_LOAD_IF2]]:
; VF1IC2-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[VEC_IV1]]
; VF1IC2-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
@@ -244,12 +242,12 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP8]] = phi i32 [ poison, %[[PRED_LOAD_CONTINUE]] ], [ [[TMP32]], %[[PRED_LOAD_IF2]] ]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF1IC2: [[PRED_STORE_IF]]:
-; VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[VEC_IV]]
+; VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP0]]
; VF1IC2-NEXT: [[TMP10:%.*]] = add nsw i32 [[VECTOR_RECUR]], [[TMP5]]
; VF1IC2-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4
; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
; VF1IC2: [[PRED_STORE_CONTINUE]]:
-; VF1IC2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
+; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
; VF1IC2: [[PRED_STORE_IF4]]:
; VF1IC2-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[VEC_IV1]]
; VF1IC2-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP5]], [[TMP8]]
@@ -260,12 +258,11 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VF1IC2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF1IC2: [[MIDDLE_BLOCK]]:
-; VF1IC2-NEXT: [[TMP14:%.*]] = xor i1 [[TMP1]], true
-; VF1IC2-NEXT: [[TMP15:%.*]] = xor i1 [[TMP2]], true
+; VF1IC2-NEXT: [[TMP15:%.*]] = xor i1 [[TMP1]], true
; VF1IC2-NEXT: [[TMP16:%.*]] = icmp eq i1 [[TMP15]], false
; VF1IC2-NEXT: [[TMP17:%.*]] = zext i1 [[TMP16]] to i64
; VF1IC2-NEXT: [[TMP18:%.*]] = add i64 1, [[TMP17]]
-; VF1IC2-NEXT: [[TMP19:%.*]] = icmp eq i1 [[TMP14]], false
+; VF1IC2-NEXT: [[TMP19:%.*]] = icmp eq i1 [[TMP15]], false
; VF1IC2-NEXT: [[TMP20:%.*]] = zext i1 [[TMP19]] to i64
; VF1IC2-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]
; VF1IC2-NEXT: [[TMP22:%.*]] = icmp ne i64 [[TMP20]], 1
@@ -513,18 +510,16 @@ define i32 @FOR_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; VF1IC2-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 33, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; VF1IC2-NEXT: [[VEC_IV:%.*]] = add i64 [[TMP0]], 0
; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
-; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
-; VF1IC2-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV1]], [[TRIP_COUNT_MINUS_1]]
+; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; VF1IC2: [[PRED_LOAD_IF]]:
-; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[VEC_IV]]
+; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP0]]
; VF1IC2-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
; VF1IC2-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; VF1IC2: [[PRED_LOAD_CONTINUE]]:
; VF1IC2-NEXT: [[TMP5:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
-; VF1IC2-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
+; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
; VF1IC2: [[PRED_LOAD_IF2]]:
; VF1IC2-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[VEC_IV1]]
; VF1IC2-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4
@@ -533,12 +528,12 @@ define i32 @FOR_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP8]] = phi i32 [ poison, %[[PRED_LOAD_CONTINUE]] ], [ [[TMP29]], %[[PRED_LOAD_IF2]] ]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF1IC2: [[PRED_STORE_IF]]:
-; VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[VEC_IV]]
+; VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP0]]
; VF1IC2-NEXT: [[TMP10:%.*]] = add nsw i32 [[VECTOR_RECUR]], [[TMP5]]
; VF1IC2-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4
; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
; VF1IC2: [[PRED_STORE_CONTINUE]]:
-; VF1IC2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
+; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
; VF1IC2: [[PRED_STORE_IF4]]:
; VF1IC2-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[VEC_IV1]]
; VF1IC2-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP5]], [[TMP8]]
@@ -549,12 +544,11 @@ define i32 @FOR_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VF1IC2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; VF1IC2: [[MIDDLE_BLOCK]]:
-; VF1IC2-NEXT: [[TMP14:%.*]] = xor i1 [[TMP1]], true
-; VF1IC2-NEXT: [[TMP15:%.*]] = xor i1 [[TMP2]], true
+; VF1IC2-NEXT: [[TMP15:%.*]] = xor i1 [[TMP1]], true
; VF1IC2-NEXT: [[TMP16:%.*]] = icmp eq i1 [[TMP15]], false
; VF1IC2-NEXT: [[TMP17:%.*]] = zext i1 [[TMP16]] to i64
; VF1IC2-NEXT: [[TMP18:%.*]] = add i64 1, [[TMP17]]
-; VF1IC2-NEXT: [[TMP19:%.*]] = icmp eq i1 [[TMP14]], false
+; VF1IC2-NEXT: [[TMP19:%.*]] = icmp eq i1 [[TMP15]], false
; VF1IC2-NEXT: [[TMP20:%.*]] = zext i1 [[TMP19]] to i64
; VF1IC2-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]
; VF1IC2-NEXT: [[TMP22:%.*]] = icmp ne i64 [[TMP20]], 1
@@ -815,18 +809,16 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; VF1IC2-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 33, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; VF1IC2-NEXT: [[VEC_IV:%.*]] = add i64 [[TMP0]], 0
; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
-; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
-; VF1IC2-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV1]], [[TRIP_COUNT_MINUS_1]]
+; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[TMP0]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; VF1IC2: [[PRED_LOAD_IF]]:
-; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[VEC_IV]]
+; VF1IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP0]]
; VF1IC2-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
; VF1IC2-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; VF1IC2: [[PRED_LOAD_CONTINUE]]:
; VF1IC2-NEXT: [[TMP5:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
-; VF1IC2-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
+; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
; VF1IC2: [[PRED_LOAD_IF2]]:
; VF1IC2-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[VEC_IV1]]
; VF1IC2-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP34]], align 4
@@ -835,12 +827,12 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP8]] = phi i32 [ poison, %[[PRED_LOAD_CONTINUE]] ], [ [[TMP35]], %[[PRED_LOAD_IF2]] ]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF1IC2: [[PRED_STORE_IF]]:
-; VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[VEC_IV]]
+; VF1IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP0]]
; VF1IC2-NEXT: [[TMP10:%.*]] = add nsw i32 [[VECTOR_RECUR]], [[TMP5]]
; VF1IC2-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4
; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
; VF1IC2: [[PRED_STORE_CONTINUE]]:
-; VF1IC2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
+; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
; VF1IC2: [[PRED_STORE_IF4]]:
; VF1IC2-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[VEC_IV1]]
; VF1IC2-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP5]], [[TMP8]]
@@ -851,12 +843,11 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VF1IC2-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; VF1IC2: [[MIDDLE_BLOCK]]:
-; VF1IC2-NEXT: [[TMP14:%.*]] = xor i1 [[TMP1]], true
-; VF1IC2-NEXT: [[TMP15:%.*]] = xor i1 [[TMP2]], true
+; VF1IC2-NEXT: [[TMP15:%.*]] = xor i1 [[TMP1]], true
; VF1IC2-NEXT: [[TMP16:%.*]] = icmp eq i1 [[TMP15]], false
; VF1IC2-NEXT: [[TMP17:%.*]] = zext i1 [[TMP16]] to i64
; VF1IC2-NEXT: [[TMP18:%.*]] = add i64 1, [[TMP17]]
-; VF1IC2-NEXT: [[TMP19:%.*]] = icmp eq i1 [[TMP14]], false
+; VF1IC2-NEXT: [[TMP19:%.*]] = icmp eq i1 [[TMP15]], false
; VF1IC2-NEXT: [[TMP20:%.*]] = zext i1 [[TMP19]] to i64
; VF1IC2-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]
; VF1IC2-NEXT: [[TMP22:%.*]] = icmp ne i64 [[TMP20]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index d319d0ffe0797..a9d0e932d2516 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2709,22 +2709,19 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[PRED_UDIV_CONTINUE4]] ]
; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_UDIV_CONTINUE4]] ]
; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
-; UNROLL-NO-VF-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
-; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = icmp ule i32 [[VEC_IV2]], [[TRIP_COUNT_MINUS_1]]
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[INDEX]], [[TRIP_COUNT_MINUS_1]]
; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; UNROLL-NO-VF: pred.udiv.if:
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = udiv i32 219220132, [[OFFSET_IDX]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]]
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.if3:
+; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4]]
+; UNROLL-NO-VF: pred.udiv.if2:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP7]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; UNROLL-NO-VF: pred.udiv.continue4:
+; UNROLL-NO-VF: pred.udiv.continue3:
; UNROLL-NO-VF-NEXT: [[TMP9]] = phi i32 [ poison, [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-VF-NEXT: [[TMP10]] = add i32 [[VEC_PHI]], [[VECTOR_RECUR]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP6]]
@@ -2733,7 +2730,7 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF25:![0-9]+]], !llvm.loop [[LOOP26:![0-9]+]]
; UNROLL-NO-VF: middle.block:
; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = select i1 [[TMP2]], i32 [[TMP10]], i32 [[VEC_PHI]]
-; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = select i1 [[TMP3]], i32 [[TMP11]], i32 [[VEC_PHI1]]
+; UNROLL-NO-VF-NEXT: [[TMP13:%.*]] = select i1 [[TMP2]], i32 [[TMP11]], i32 [[VEC_PHI1]]
; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP12]]
; UNROLL-NO-VF-NEXT: br label [[BB2:%.*]]
; UNROLL-NO-VF: bb1:
@@ -3033,17 +3030,14 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_STORE_CONTINUE6]] ]
; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
-; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
-; UNROLL-NO-VF-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
-; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = icmp ule i32 [[VEC_IV2]], [[TRIP_COUNT_MINUS_1]]
+; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = icmp ule i32 [[INDEX]], [[TRIP_COUNT_MINUS_1]]
; UNROLL-NO-VF-NEXT: br i1 [[TMP4]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; UNROLL-NO-VF: pred.udiv.if:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[OFFSET_IDX]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE]]
; UNROLL-NO-VF: pred.udiv.continue:
; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
-; UNROLL-NO-VF-NEXT: br i1 [[TMP5]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP4]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; UNROLL-NO-VF: pred.udiv.if2:
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = udiv i32 219220132, [[TMP3]]
; UNROLL-NO-VF-NEXT: br label [[PRED_UDIV_CONTINUE4]]
@@ -3053,12 +3047,13 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP7]]
; UNROLL-NO-VF-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; UNROLL-NO-VF: pred.store.if:
-; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[VEC_IV]]
+; UNROLL-NO-VF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
; UNROLL-NO-VF-NEXT: store i32 [[OFFSET_IDX]], ptr [[TMP12]], align 4
; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE]]
; UNROLL-NO-VF: pred.store.continue:
-; UNROLL-NO-VF-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE6]]
; UNROLL-NO-VF: pred.store.if4:
+; UNROLL-NO-VF-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX]], 1
; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[VEC_IV2]]
; UNROLL-NO-VF-NEXT: store i32 [[TMP3]], ptr [[TMP15]], align 4
; UNROLL-NO-VF-NEXT: br label [[PRED_STORE_CONTINUE6]]
@@ -3068,7 +3063,7 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-VF-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF25]], !llvm.loop [[LOOP28:![0-9]+]]
; UNROLL-NO-VF: middle.block:
; UNROLL-NO-VF-NEXT: [[TMP16:%.*]] = select i1 [[TMP4]], i32 [[TMP10]], i32 [[VEC_PHI]]
-; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = select i1 [[TMP5]], i32 [[TMP11]], i32 [[VEC_PHI1]]
+; UNROLL-NO-VF-NEXT: [[TMP17:%.*]] = select i1 [[TMP4]], i32 [[TMP11]], i32 [[VEC_PHI1]]
; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP17]], [[TMP16]]
; UNROLL-NO-VF-NEXT: br label [[BB2:%.*]]
; UNROLL-NO-VF: bb1:
diff --git a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
index c07134229322e..5d2218a9a0276 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
@@ -118,33 +118,30 @@ define void @pr45679(ptr %A) {
; VF1UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; VF1UF4: vector.body:
; VF1UF4-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
-; VF1UF4-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
-; VF1UF4-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
-; VF1UF4-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; VF1UF4-NEXT: [[TMP4:%.*]] = icmp ule i32 [[INDEX]], 13
-; VF1UF4-NEXT: [[TMP5:%.*]] = icmp ule i32 [[TMP1]], 13
-; VF1UF4-NEXT: [[TMP6:%.*]] = icmp ule i32 [[TMP2]], 13
-; VF1UF4-NEXT: [[TMP7:%.*]] = icmp ule i32 [[TMP3]], 13
; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; VF1UF4: pred.store.if:
; VF1UF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]
; VF1UF4-NEXT: store i32 13, ptr [[TMP8]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE]]
; VF1UF4: pred.store.continue:
-; VF1UF4-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
; VF1UF4: pred.store.if1:
+; VF1UF4-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; VF1UF4-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP1]]
; VF1UF4-NEXT: store i32 13, ptr [[TMP9]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE2]]
; VF1UF4: pred.store.continue2:
-; VF1UF4-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
; VF1UF4: pred.store.if3:
+; VF1UF4-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; VF1UF4-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP2]]
; VF1UF4-NEXT: store i32 13, ptr [[TMP10]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE4]]
; VF1UF4: pred.store.continue4:
-; VF1UF4-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
; VF1UF4: pred.store.if5:
+; VF1UF4-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; VF1UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP3]]
; VF1UF4-NEXT: store i32 13, ptr [[TMP11]], align 1
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE6]]
@@ -288,13 +285,7 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
; VF1UF4-NEXT: br label [[VECTOR_BODY:%.*]]
; VF1UF4: vector.body:
; VF1UF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
-; VF1UF4-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; VF1UF4-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; VF1UF4-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; VF1UF4-NEXT: [[TMP4:%.*]] = icmp ule i64 [[INDEX]], 13
-; VF1UF4-NEXT: [[TMP5:%.*]] = icmp ule i64 [[TMP1]], 13
-; VF1UF4-NEXT: [[TMP6:%.*]] = icmp ule i64 [[TMP2]], 13
-; VF1UF4-NEXT: [[TMP7:%.*]] = icmp ule i64 [[TMP3]], 13
; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; VF1UF4: pred.store.if:
; VF1UF4-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
@@ -302,22 +293,25 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
; VF1UF4-NEXT: store i64 [[TMP9]], ptr [[B:%.*]], align 8
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE]]
; VF1UF4: pred.store.continue:
-; VF1UF4-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
; VF1UF4: pred.store.if1:
+; VF1UF4-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; VF1UF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
; VF1UF4-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP11]], align 8
; VF1UF4-NEXT: store i64 [[TMP12]], ptr [[B]], align 8
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE2]]
; VF1UF4: pred.store.continue2:
-; VF1UF4-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
; VF1UF4: pred.store.if3:
+; VF1UF4-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; VF1UF4-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
; VF1UF4-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP14]], align 8
; VF1UF4-NEXT: store i64 [[TMP15]], ptr [[B]], align 8
; VF1UF4-NEXT: br label [[PRED_STORE_CONTINUE4]]
; VF1UF4: pred.store.continue4:
-; VF1UF4-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; VF1UF4-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
; VF1UF4: pred.store.if5:
+; VF1UF4-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; VF1UF4-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
; VF1UF4-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP17]], align 8
; VF1UF4-NEXT: store i64 [[TMP18]], ptr [[B]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
index 959aa01edde92..7cedd12aee5cd 100644
--- a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
@@ -11,13 +11,10 @@ define float @pr70988() {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX1]], 0
-; CHECK-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX1]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 1020
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i32 [[VEC_IV2]], 1020
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[INDEX1]], 1020
; CHECK-NEXT: [[TMP2:%.*]] = select contract i1 [[TMP0]], float 1.000000e+00, float -0.000000e+00
; CHECK-NEXT: [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = select contract i1 [[TMP1]], float 1.000000e+00, float -0.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = select contract i1 [[TMP0]], float 1.000000e+00, float -0.000000e+00
; CHECK-NEXT: [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i32 [[INDEX1]], 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 1022
@@ -35,12 +32,10 @@ define float @pr70988() {
; CHECK-ALM: vector.body:
; CHECK-ALM-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX1]], 1
; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[INDEX1]], 1021
-; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = icmp ult i32 [[TMP1]], 1021
; CHECK-ALM-NEXT: [[TMP2:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float 1.000000e+00, float -0.000000e+00
; CHECK-ALM-NEXT: [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]]
-; CHECK-ALM-NEXT: [[TMP4:%.*]] = select contract i1 [[ACTIVE_LANE_MASK2]], float 1.000000e+00, float -0.000000e+00
+; CHECK-ALM-NEXT: [[TMP4:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float 1.000000e+00, float -0.000000e+00
; CHECK-ALM-NEXT: [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]]
; CHECK-ALM-NEXT: [[INDEX_NEXT3]] = add nuw i32 [[INDEX1]], 2
; CHECK-ALM-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 1022
@@ -76,19 +71,17 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE3:%.*]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_LOAD_CONTINUE3]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
-; CHECK-NEXT: [[VEC_IV1:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 14
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i32 [[VEC_IV1]], 14
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[INDEX]], 14
; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[SRC]], i32 [[VEC_IV]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[SRC]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK: pred.load.continue:
; CHECK-NEXT: [[TMP5:%.*]] = phi float [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]]
+; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]]
; CHECK: pred.load.if1:
+; CHECK-NEXT: [[VEC_IV1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[SRC]], i32 [[VEC_IV1]]
; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]]
@@ -96,7 +89,7 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
; CHECK-NEXT: [[TMP9:%.*]] = phi float [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP8]], [[PRED_LOAD_IF2]] ]
; CHECK-NEXT: [[TMP10:%.*]] = select contract i1 [[TMP0]], float [[TMP5]], float -0.000000e+00
; CHECK-NEXT: [[TMP11:%.*]] = fadd contract float [[VEC_PHI]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = select contract i1 [[TMP1]], float [[TMP9]], float -0.000000e+00
+; CHECK-NEXT: [[TMP12:%.*]] = select contract i1 [[TMP0]], float [[TMP9]], float -0.000000e+00
; CHECK-NEXT: [[TMP13]] = fadd contract float [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
@@ -115,9 +108,7 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
; CHECK-ALM: vector.body:
; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE3:%.*]] ]
; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_LOAD_CONTINUE3]] ]
-; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[INDEX]], 15
-; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = icmp ult i32 [[TMP1]], 15
; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK-ALM: pred.load.if:
; CHECK-ALM-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[SRC]], i32 [[INDEX]]
@@ -125,16 +116,17 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
; CHECK-ALM-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK-ALM: pred.load.continue:
; CHECK-ALM-NEXT: [[TMP4:%.*]] = phi float [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ]
-; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK1]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]]
-; CHECK-ALM: pred.load.if2:
+; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]]
+; CHECK-ALM: pred.load.if1:
+; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-ALM-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP1]]
; CHECK-ALM-NEXT: [[TMP6:%.*]] = load float, ptr [[TMP5]], align 4
; CHECK-ALM-NEXT: br label [[PRED_LOAD_CONTINUE3]]
-; CHECK-ALM: pred.load.continue3:
+; CHECK-ALM: pred.load.continue2:
; CHECK-ALM-NEXT: [[TMP7:%.*]] = phi float [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP6]], [[PRED_LOAD_IF2]] ]
; CHECK-ALM-NEXT: [[TMP8:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float [[TMP4]], float -0.000000e+00
; CHECK-ALM-NEXT: [[TMP9:%.*]] = fadd contract float [[VEC_PHI]], [[TMP8]]
-; CHECK-ALM-NEXT: [[TMP10:%.*]] = select contract i1 [[ACTIVE_LANE_MASK1]], float [[TMP7]], float -0.000000e+00
+; CHECK-ALM-NEXT: [[TMP10:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float [[TMP7]], float -0.000000e+00
; CHECK-ALM-NEXT: [[TMP11]] = fadd contract float [[TMP9]], [[TMP10]]
; CHECK-ALM-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-ALM-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
@@ -172,13 +164,10 @@ define float @fadd_reduction_with_live_in(float %inc) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
-; CHECK-NEXT: [[VEC_IV1:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 1000
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i32 [[VEC_IV1]], 1000
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[INDEX]], 1000
; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP0]], float [[INC]], float -0.000000e+00
; CHECK-NEXT: [[TMP3:%.*]] = fadd float [[VEC_PHI]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP1]], float [[INC]], float -0.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP0]], float [[INC]], float -0.000000e+00
; CHECK-NEXT: [[TMP5]] = fadd float [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1002
@@ -197,12 +186,10 @@ define float @fadd_reduction_with_live_in(float %inc) {
; CHECK-ALM: vector.body:
; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[INDEX]], 1001
-; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = icmp ult i32 [[TMP1]], 1001
; CHECK-ALM-NEXT: [[TMP2:%.*]] = select i1 [[ACTIVE_LANE_MASK]], float [[INC]], float -0.000000e+00
; CHECK-ALM-NEXT: [[TMP3:%.*]] = fadd float [[VEC_PHI]], [[TMP2]]
-; CHECK-ALM-NEXT: [[TMP4:%.*]] = select i1 [[ACTIVE_LANE_MASK1]], float [[INC]], float -0.000000e+00
+; CHECK-ALM-NEXT: [[TMP4:%.*]] = select i1 [[ACTIVE_LANE_MASK]], float [[INC]], float -0.000000e+00
; CHECK-ALM-NEXT: [[TMP5]] = fadd float [[TMP3]], [[TMP4]]
; CHECK-ALM-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-ALM-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1002
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index 57e0e9b63b8de..35aa33fef9a3c 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -17,33 +17,30 @@ define void @VF1-VPlanExe(ptr %dst) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = icmp ule i64 [[INDEX]], 14
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ule i64 [[TMP1]], 14
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ule i64 [[TMP2]], 14
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ule i64 [[TMP3]], 14
; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store i32 0, ptr [[TMP8]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
-; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
; CHECK: pred.store.if1:
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT: store i32 0, ptr [[TMP9]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]
; CHECK: pred.store.continue2:
-; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
; CHECK: pred.store.if3:
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]
; CHECK-NEXT: store i32 0, ptr [[TMP10]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]
; CHECK: pred.store.continue4:
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
; CHECK: pred.store.if5:
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
; CHECK-NEXT: store i32 0, ptr [[TMP11]], align 4
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]
@@ -90,34 +87,27 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]
-; CHECK-NEXT: [[VEC_IV:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT: [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IV6:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[VEC_IV]], 14
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV4]], 14
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV5]], 14
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[INDEX]], 14
; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
-; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
-; CHECK: pred.store.if7:
+; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
+; CHECK: pred.store.if4:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP1]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]
-; CHECK: pred.store.continue8:
-; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; CHECK: pred.store.if9:
+; CHECK: pred.store.continue5:
+; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
+; CHECK: pred.store.if6:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP2]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]
-; CHECK: pred.store.continue10:
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.if11:
+; CHECK: pred.store.continue7:
+; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE12]]
+; CHECK: pred.store.if8:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]
-; CHECK: pred.store.continue12:
+; CHECK: pred.store.continue9:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
More information about the llvm-commits
mailing list