[llvm] [LAA] Avoid stray predicates from replaceSymStrides (PR #216350)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 07:53:22 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/216350
>From ca896460b1d45b2178a888b80dff41b0d4d6da0f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 28 Sep 2026 11:41:13 +0100
Subject: [PATCH 1/3] [LAA] Pre-commit symbolic-stride test
---
.../LoopAccessAnalysis/symbolic-stride.ll | 62 +++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
index f4aff75582c02..d3f8261d3032e 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
@@ -918,3 +918,65 @@ loop:
exit:
ret void
}
+
+define void @dropped_predicates(ptr %a, ptr %b, i64 %c, i64 %d, i64 %n) {
+; CHECK-LABEL: 'dropped_predicates'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.b.mul = getelementptr i32, ptr %b, i64 %mul
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.a.idx.1 = getelementptr { i32, i32 }, ptr %a, i64 %idx, i32 1
+; CHECK-NEXT: %gep.a.idx.0 = getelementptr { i32, i32 }, ptr %a, i64 %idx, i32 0
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %b High: ((4 * %n) + %b))
+; CHECK-NEXT: Member: {%b,+,4}<%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (8 + %a) High: (8 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(12 + %a),+,8}<%loop>
+; CHECK-NEXT: Member: {(8 + %a),+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-NEXT: Equal predicate: %c == 1
+; CHECK-NEXT: Equal predicate: %d == 1
+; CHECK-NEXT: {(8 + %a),+,8}<%loop> Added Flags: <nusw>
+; CHECK-NEXT: {(12 + %a),+,8}<%loop> Added Flags: <nusw>
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+; CHECK-NEXT: [PSE] %gep.b.mul = getelementptr i32, ptr %b, i64 %mul:
+; CHECK-NEXT: {%b,+,(4 * %d)}<%loop>
+; CHECK-NEXT: --> {%b,+,4}<%loop>
+; CHECK-NEXT: [PSE] %gep.a.idx.1 = getelementptr { i32, i32 }, ptr %a, i64 %idx, i32 1:
+; CHECK-NEXT: {(4 + (8 * %d) + %a),+,(8 * %c)}<%loop>
+; CHECK-NEXT: --> {(12 + %a),+,8}<%loop>
+; CHECK-NEXT: [PSE] %gep.a.idx.0 = getelementptr { i32, i32 }, ptr %a, i64 %idx, i32 0:
+; CHECK-NEXT: {((8 * %d) + %a),+,(8 * %c)}<%loop>
+; CHECK-NEXT: --> {(8 + %a),+,8}<%loop>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %mul = mul i64 %iv, %d
+ %gep.b.mul = getelementptr i32, ptr %b, i64 %mul
+ %ld.b = load i32, ptr %gep.b.mul
+ %mul.iv.s = mul i64 %iv, %c
+ %idx = add i64 %mul.iv.s, %d
+ %gep.a.idx.1 = getelementptr {i32, i32}, ptr %a, i64 %idx, i32 1
+ %ld.a = load i32, ptr %gep.a.idx.1
+ %v = add i32 %ld.a, %ld.b
+ %gep.a.idx.0 = getelementptr {i32, i32}, ptr %a, i64 %idx, i32 0
+ store i32 %v, ptr %gep.a.idx.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 2c7f5a40d8d14f88233ebb01e12e8763262a6bb5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 14 Aug 2026 17:08:57 +0100
Subject: [PATCH 2/3] [LAA] Avoid stray predicates from replaceSymStrides
To avoid stray predicates being added from the use of
replaceSymbolicStridesSCEV, simply add an optional Predicates argument
to be filled in when the caller doesn't want fresh predicates to be
added to PSE directly. This allows us to drop some stray predicates when
vectorizing.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 11 +-
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 33 ++-
llvm/lib/Analysis/VectorUtils.cpp | 2 +-
.../version-stride-with-integer-casts.ll | 52 ++--
.../LoopVectorize/vplan-based-stride-mv.ll | 222 +++++++-----------
5 files changed, 134 insertions(+), 186 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 0e5f5a1e81b77..7aeb24e07ddb4 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -897,13 +897,16 @@ class LoopAccessInfo {
/// \p PSE is true.
///
/// If necessary this method will version the stride of the pointer according
-/// to \p PtrToStride and therefore add further predicates to \p PSE.
+/// to \p PtrToStride and therefore add further predicates to \p PSE, except
+/// when \p Predicates is given, in which case, it adds the fresh predicates
+/// there instead of to \p PSE directly.
///
/// \p PtrToStride provides the mapping between the pointer value and its
/// stride as collected by LoopVectorizationLegality::collectStridedAccess.
-LLVM_ABI const SCEV *
-replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
- const SymbolicStrideMap &PtrToStride, Value *Ptr);
+LLVM_ABI const SCEV *replaceSymbolicStrideSCEV(
+ PredicatedScalarEvolution &PSE, const Loop *Lp,
+ const SymbolicStrideMap &PtrToStride, Value *Ptr,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
/// If \p AR is an affine AddRec for \p Lp with a constant step, return the
/// step in units of \p AccessTy's allocation size. Returns std::nullopt if the
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 9c77a78336de2..75d5fb97030d6 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -153,10 +153,10 @@ bool VectorizerParams::isInterleaveForced() {
return ::VectorizationInterleave.getNumOccurrences() > 0;
}
-const SCEV *
-llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
- const SymbolicStrideMap &PtrToStride,
- Value *Ptr) {
+const SCEV *llvm::replaceSymbolicStrideSCEV(
+ PredicatedScalarEvolution &PSE, const Loop *Lp,
+ const SymbolicStrideMap &PtrToStride, Value *Ptr,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates) {
const SCEV *OrigSCEV = PSE.getSCEV(Ptr);
// If there is an entry in the map return the SCEV of the pointer with the
@@ -168,11 +168,18 @@ llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
ScalarEvolution *SE = PSE.getSE();
const SCEV *CT = SE->getOne(StrideSCEV->getType());
- PSE.addPredicate(*SE->getEqualPredicate(StrideSCEV, CT));
- const SCEV *Expr = PSE.getSCEV(Ptr);
-
- LLVM_DEBUG(dbgs() << "LAA: Replacing SCEV: " << *OrigSCEV
- << " by: " << *Expr << "\n");
+ const SCEV *Expr;
+ const SCEVPredicate *EqPred = SE->getEqualPredicate(StrideSCEV, CT);
+ if (Predicates) {
+ Predicates->push_back(EqPred);
+ Expr = SE->rewriteUsingPredicate(OrigSCEV, Lp,
+ SCEVUnionPredicate(*Predicates, *SE));
+ } else {
+ PSE.addPredicate(*EqPred);
+ Expr = PSE.getSCEV(Ptr);
+ }
+ LLVM_DEBUG(dbgs() << "LAA: Replacing SCEV: " << *OrigSCEV << " by: " << *Expr
+ << "\n");
return Expr;
}
@@ -1374,7 +1381,8 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
for (const auto &[Idx, Q] : enumerate(RTCheckPtrs)) dbgs()
<< "\t(" << Idx << ") " << *Q.getPointer() << "\n");
} else {
- RTCheckPtrs = {{replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr), false}};
+ RTCheckPtrs = {
+ {replaceSymbolicStrideSCEV(PSE, TheLoop, StridesMap, Ptr), false}};
}
/// Check whether all pointers can participate in a runtime bounds check. They
@@ -1406,7 +1414,7 @@ bool AccessAnalysis::createCheckForAccess(RuntimePointerChecking &RtCheck,
PSE.addPredicates(Predicates);
Predicates.clear();
if (auto *StrideAR = dyn_cast<SCEVAddRecExpr>(
- replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr)))
+ replaceSymbolicStrideSCEV(PSE, TheLoop, StridesMap, Ptr)))
AR = StrideAR;
P.setPointer(AR);
}
@@ -1758,7 +1766,8 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
const Loop *Lp, const DominatorTree &DT,
const SymbolicStrideMap &StridesMap, bool ShouldCheckWrap,
SmallVectorImpl<const SCEVPredicate *> *Predicates) {
- const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr);
+ const SCEV *PtrScev =
+ replaceSymbolicStrideSCEV(PSE, Lp, StridesMap, Ptr, Predicates);
if (PSE.getSE()->isLoopInvariant(PtrScev, Lp))
return 0;
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index c1d6f725dbdef..90ee02d9526c6 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1330,7 +1330,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses(
/*ShouldCheckWrap=*/false, Predicates)
.value_or(0);
- const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
+ const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, TheLoop, Strides, Ptr);
AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size,
getLoadStoreAlignment(&I));
}
diff --git a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
index 35d92f139a41e..3a57803775a5e 100644
--- a/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/version-stride-with-integer-casts.ll
@@ -146,43 +146,35 @@ define void @versioned_sext_use_in_gep(i32 %scale, ptr %dst, i64 %scale.2) {
; CHECK-SAME: i32 [[SCALE:%.*]], ptr [[DST:%.*]], i64 [[SCALE_2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SCALE_EXT:%.*]] = sext i32 [[SCALE]] to i64
-; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[SCALE]], 1
-; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP83:%.*]] = getelementptr i8, ptr [[DST]], i64 [[SCALE_2]]
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[SCALE_EXT]], [[SCALE_2]]
+; CHECK-NEXT: [[TMP83:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[SCALE_EXT]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP16]]
; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP11]], align 8
; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP13]], align 8
; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP15]], align 8
; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP17]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_MUL:%.*]] = mul i64 [[IV]], [[SCALE_EXT]]
-; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_MUL]]
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[SCALE_MUL:%.*]] = mul i64 [[SCALE_EXT]], [[SCALE_2]]
-; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[SCALE_MUL]]
-; CHECK-NEXT: store ptr [[GEP_2]], ptr [[GEP_1]], align 8
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 256
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -247,7 +239,7 @@ define void @test_versioned_with_different_uses(i32 %offset, ptr noalias %dst.1,
; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP12]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -265,7 +257,7 @@ define void @test_versioned_with_different_uses(i32 %offset, ptr noalias %dst.1,
; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], [[OFFSET_EXT]]
; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_3]], 200
-; CHECK-NEXT: br i1 [[EC]], label %[[OUTER_HEADER_LOOPEXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[OUTER_HEADER_LOOPEXIT]], label %[[INNER_LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -317,7 +309,7 @@ define void @test_versioned_with_non_ex_use(i32 %offset, ptr noalias %dst.1, ptr
; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP20]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -336,7 +328,7 @@ define void @test_versioned_with_non_ex_use(i32 %offset, ptr noalias %dst.1, ptr
; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], 1
; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_3]], 200
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -384,7 +376,7 @@ define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
; CHECK-NEXT: store <4 x i16> splat (i16 1), ptr [[TMP4]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -395,7 +387,7 @@ define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {
; CHECK-NEXT: store i16 [[G_16]], ptr [[GEP]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], [[G_64]]
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 16
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -491,7 +483,7 @@ define void @test_scale_inside_and_outside_cast(ptr noalias %A, ptr noalias %B,
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -555,7 +547,7 @@ define void @test_scale_split_across_cast(ptr noalias %A, ptr noalias %B, i32 %s
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 30101a743b5d0..1177c095c9188 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -677,36 +677,24 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
; COMPARE-LAA-MV: [[HEADER1]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
-; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -781,7 +769,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -860,7 +848,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -877,7 +865,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -961,7 +949,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -979,7 +967,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1064,7 +1052,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1083,7 +1071,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1171,7 +1159,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1190,7 +1178,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1258,13 +1246,13 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P0]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-NEXT: [[WIDE_GEP1:%.*]] = getelementptr i8, ptr [[P1]], <4 x i64> [[TMP0]]
@@ -1274,28 +1262,12 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
; COMPARE-LAA-MV: [[HEADER1]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
-; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP23:![0-9]+]]
-; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1362,13 +1334,13 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[P1]], <4 x i64> [[TMP0]]
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = sext <4 x i32> [[TMP15]] to <4 x i64>
@@ -1378,28 +1350,12 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
; COMPARE-LAA-MV: [[HEADER1]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
-; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
-; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP25:![0-9]+]]
-; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1482,7 +1438,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1499,7 +1455,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1577,7 +1533,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1593,7 +1549,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1670,7 +1626,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1685,7 +1641,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1763,7 +1719,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -1845,7 +1801,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -1947,7 +1903,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1963,7 +1919,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2232,7 +2188,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2248,7 +2204,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2348,7 +2304,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2435,7 +2391,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2558,7 +2514,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2638,7 +2594,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2713,7 +2669,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2727,7 +2683,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2787,36 +2743,24 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, <4 x i64> [[TMP0]], i64 42
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
; COMPARE-LAA-MV: [[HEADER1]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[IDX]], i64 42
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP45:![0-9]+]]
-; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -2887,7 +2831,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2960,7 +2904,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2975,7 +2919,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3047,7 +2991,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX1]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH1]]:
@@ -3062,7 +3006,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP46:![0-9]+]]
; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3143,7 +3087,7 @@ define void @trunc_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stri
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[TMP31]], ptr [[TMP32]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP33]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP33]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH1]]:
@@ -3161,7 +3105,7 @@ define void @trunc_stride_extra_narrow_use(ptr noalias %p.out, ptr %p, i64 %stri
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP52:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[HEADER]], !llvm.loop [[LOOP48:![0-9]+]]
; COMPARE-LAA-MV: [[HEADER]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3255,7 +3199,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3275,7 +3219,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-LAA-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP50:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3362,7 +3306,7 @@ define void @trunc_ext_chained_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
@@ -3445,7 +3389,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
; COMPARE-LAA-MV: [[PRED_STORE_IF1]]:
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE2:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF3]]:
@@ -3464,7 +3408,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE6]]:
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_CONTINUE4]], label %[[PRED_STORE_CONTINUE2]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[PRED_STORE_CONTINUE4]], label %[[PRED_STORE_CONTINUE2]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE2]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3537,7 +3481,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -3609,7 +3553,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3623,7 +3567,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3701,7 +3645,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3717,7 +3661,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3795,7 +3739,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3810,7 +3754,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP64:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3889,7 +3833,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP65:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3905,7 +3849,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP66:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4062,7 +4006,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 4
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP67:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4077,7 +4021,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-LAA-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP68:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP64:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4194,7 +4138,7 @@ define void @stride_mv_predicated_btc(ptr noalias %p.out, ptr %p, i32 %M, i64 %s
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP69:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP65:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -4212,7 +4156,7 @@ define void @stride_mv_predicated_btc(ptr noalias %p.out, ptr %p, i32 %M, i64 %s
; COMPARE-LAA-MV-NEXT: [[I_NEXT]] = add i16 [[I]], 1
; COMPARE-LAA-MV-NEXT: [[I_NEXT_EXT:%.*]] = sext i16 [[I_NEXT]] to i32
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp sle i32 [[I_NEXT_EXT]], [[M]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP70:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP66:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4347,7 +4291,7 @@ define void @known_non_unit_via_assume(ptr noalias %out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP71:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP67:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4361,7 +4305,7 @@ define void @known_non_unit_via_assume(ptr noalias %out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[GEP_OUT]], align 4
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP72:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP68:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4416,7 +4360,7 @@ define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p, ptr %stride
; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_load_range(
; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], ptr [[STRIDE_PTR:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[VECTOR_PH:.*]]:
-; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = load i64, ptr [[STRIDE_PTR]], align 8, !range [[RNG73:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = load i64, ptr [[STRIDE_PTR]], align 8, !range [[RNG69:![0-9]+]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
>From e2b4f0402728c6637afb22375d3d0fdac6d3b8b5 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 1 Oct 2026 15:46:08 +0100
Subject: [PATCH 3/3] [LAA] Fix NFC nits
---
llvm/include/llvm/Analysis/LoopAccessAnalysis.h | 4 ++--
llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 7aeb24e07ddb4..2bb85fef7cb31 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -898,8 +898,8 @@ class LoopAccessInfo {
///
/// If necessary this method will version the stride of the pointer according
/// to \p PtrToStride and therefore add further predicates to \p PSE, except
-/// when \p Predicates is given, in which case, it adds the fresh predicates
-/// there instead of to \p PSE directly.
+/// when \p Predicates is given, in which case, it adds predicates there instead
+/// of to \p PSE directly.
///
/// \p PtrToStride provides the mapping between the pointer value and its
/// stride as collected by LoopVectorizationLegality::collectStridedAccess.
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
index d3f8261d3032e..d5781d1529b22 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
@@ -919,8 +919,8 @@ exit:
ret void
}
-define void @dropped_predicates(ptr %a, ptr %b, i64 %c, i64 %d, i64 %n) {
-; CHECK-LABEL: 'dropped_predicates'
+define void @stray_predicates(ptr %a, ptr %b, i64 %c, i64 %d, i64 %n) {
+; CHECK-LABEL: 'stray_predicates'
; CHECK-NEXT: loop:
; CHECK-NEXT: Memory dependences are safe with run-time checks
; CHECK-NEXT: Dependences:
More information about the llvm-commits
mailing list