[llvm] [LV] Rework collectUnitStridePredicates (PR #216362)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 01:05:12 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/216362
>From d9d4301d2d0dc50383216c9a6f8106f854bb561f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 14 Aug 2026 13:07:11 +0100
Subject: [PATCH 1/2] [LV] Rework collectUnitStridePredicates
The intent of collectUnitStridePredicates is to add unit-strided
predicates from LoopAccessAnalysis' stride-speculation to PSE, so that
VPlanTransforms::replaceSymbolicStrides works. It is, however, doing
this in a poor and inaccurate fashion by again calling
llvm::getPtrStride, when the symbolic strides map is guaranteed to have
StrideSCEVs speculated to unit. Rewrite it completely, simply adding the
hard-coded equals-unit predicate to each StrideSCEV in the stride map.
The patch has necessitated making deterministic the order of entries in
the symbolic strides map.
---
.../llvm/Analysis/LoopAccessAnalysis.h | 2 +-
.../Vectorize/LoopVectorizationLegality.h | 8 +-
.../Vectorize/LoopVectorizationLegality.cpp | 16 +--
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
.../Transforms/LoopVectorize/induction.ll | 40 +++---
...d-inductions-vs-first-order-recurrences.ll | 114 +++++++++++++++---
.../LoopVectorize/predicated-inductions.ll | 51 ++++++--
7 files changed, 175 insertions(+), 60 deletions(-)
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 792b8ba9cabc5..b68728fbad859 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -57,7 +57,7 @@ struct VectorizerParams {
/// Maps a pointer to its symbolic (non-constant) stride. Strides are loop
/// invariant, which collectStridedAccess checks before inserting.
-using SymbolicStrideMap = DenseMap<Value *, const SCEVUnknown *>;
+using SymbolicStrideMap = MapVector<Value *, const SCEVUnknown *>;
/// Checks memory dependences among accesses to the same underlying
/// object to determine whether there vectorization is legal or not (and at
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 7b8b27c6541e1..336aad971765f 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -367,9 +367,11 @@ class LoopVectorizationLegality {
/// to be vectorized.
LLVM_ABI bool blockNeedsPredication(const BasicBlock *BB) const;
- /// Add unit stride predicates for memory accesses to PSE, if runtime checks
- /// are allowed and an inner loop is vectorized.
- LLVM_ABI void collectUnitStridePredicates() const;
+ /// Get the unit-stride predicates from LoopAccessAnalysis'
+ /// stride-speculation, if runtime checks are allowed and an inner loop is
+ /// vectorized.
+ LLVM_ABI SmallVector<const SCEVPredicate *>
+ collectUnitStridePredicates() const;
/// Check if this pointer is consecutive when vectorizing. This happens
/// when the last index of the GEP is the induction variable, or that the
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0c080d9434ea8..df11571314236 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -442,14 +442,16 @@ static bool storeToSameAddress(ScalarEvolution *SE, StoreInst *A,
return SE->getSCEV(APtr) == SE->getSCEV(BPtr);
}
-void LoopVectorizationLegality::collectUnitStridePredicates() const {
+SmallVector<const SCEVPredicate *>
+LoopVectorizationLegality::collectUnitStridePredicates() const {
+ ScalarEvolution &SE = *PSE.getSE();
+ SmallVector<const SCEVPredicate *> Predicates;
if (!AllowRuntimeSCEVChecks || !TheLoop->isInnermost())
- return;
-
- for (BasicBlock *BB : TheLoop->blocks())
- for (Instruction &I : *BB)
- if (Value *Ptr = getLoadStorePointerOperand(&I))
- isConsecutivePtr(getLoadStoreType(&I), Ptr);
+ return {};
+ for (const SCEV *StrideSCEV : LAI->getSymbolicStrides().values())
+ Predicates.push_back(
+ SE.getEqualPredicate(StrideSCEV, SE.getOne(StrideSCEV->getType())));
+ return Predicates;
}
int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 716c73a7c63b3..feb66a645a86f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5290,8 +5290,8 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// Cases that may be vectorized may be optimized by unit stride predicates.
// TODO: Currently unit stride predicates are added unconditionally, even if
// they are not used for the selected VF (e.g. when only interleaving).
- if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
- Legal->collectUnitStridePredicates();
+ if (MaxFactors.hasVector())
+ PSE.addPredicates(Legal->collectUnitStridePredicates());
auto VPlan1 = tryToBuildVPlan1();
if (!VPlan1)
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 1683f784daf75..47c5a045257bf 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -3330,11 +3330,11 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; CHECK: vector.scevcheck:
; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
; CHECK-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP12]]
@@ -3397,11 +3397,11 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; IND: vector.scevcheck:
; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; IND-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; IND-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; IND-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; IND-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
; IND-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; IND-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; IND-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; IND-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; IND-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; IND-NEXT: [[TMP10:%.*]] = or i1 [[TMP5]], [[TMP9]]
@@ -3464,11 +3464,11 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; UNROLL: vector.scevcheck:
; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; UNROLL-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; UNROLL-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; UNROLL-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; UNROLL-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
@@ -3534,11 +3534,11 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; UNROLL-NO-IC: vector.scevcheck:
; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = or i1 [[TMP7]], [[TMP8]]
; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP12]]
@@ -3604,11 +3604,11 @@ define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) {
; INTERLEAVE: vector.scevcheck:
; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
; INTERLEAVE-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; INTERLEAVE-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; INTERLEAVE-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
@@ -3706,11 +3706,11 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; CHECK: vector.scevcheck:
; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
; CHECK-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP13]]
@@ -3776,11 +3776,11 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; IND: vector.scevcheck:
; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; IND-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; IND-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; IND-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; IND-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
; IND-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; IND-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; IND-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; IND-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; IND-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; IND-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
@@ -3846,11 +3846,11 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; UNROLL: vector.scevcheck:
; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; UNROLL-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; UNROLL-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; UNROLL-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; UNROLL-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
@@ -3919,11 +3919,11 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; UNROLL-NO-IC: vector.scevcheck:
; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP3]], [[TMP9]]
; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = or i1 [[TMP7]], [[TMP8]]
; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP11]], [[TMP13]]
@@ -3992,11 +3992,11 @@ define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) {
; INTERLEAVE: vector.scevcheck:
; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8
; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]]
-; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP2]], [[T]]
+; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]]
; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255
; INTERLEAVE-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; INTERLEAVE-NEXT: [[TMP6:%.*]] = add i8 [[T]], [[TMP1]]
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], [[T]]
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp slt i8 [[TMP6]], [[T]]
; INTERLEAVE-NEXT: [[TMP8:%.*]] = icmp ugt i32 [[LEN]], 255
; INTERLEAVE-NEXT: [[TMP10:%.*]] = or i1 [[TMP7]], [[TMP8]]
; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll b/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
index 0f6ba95f9aee3..97a29c2bd68c0 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-inductions-vs-first-order-recurrences.ll
@@ -459,20 +459,57 @@ define i64 @for_and_ind_liveout_gep(ptr %dst, i64 %n) {
; THRESHOLD-LABEL: define i64 @for_and_ind_liveout_gep(
; THRESHOLD-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; THRESHOLD-NEXT: [[ENTRY:.*]]:
+; THRESHOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; THRESHOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; THRESHOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; THRESHOLD: [[VECTOR_SCEVCHECK]]:
+; THRESHOLD-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; THRESHOLD-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX]], -1
+; THRESHOLD-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[TMP1]], 65535
+; THRESHOLD-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; THRESHOLD: [[VECTOR_PH]]:
+; THRESHOLD-NEXT: [[TMP3:%.*]] = and i64 [[TMP0]], 3
+; THRESHOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP3]]
+; THRESHOLD-NEXT: [[TMP4:%.*]] = trunc i64 [[N_VEC]] to i16
; THRESHOLD-NEXT: br label %[[LOOP:.*]]
; THRESHOLD: [[LOOP]]:
-; THRESHOLD-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; THRESHOLD-NEXT: [[NARROW:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
-; THRESHOLD-NEXT: [[PREV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[EXT:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[VEC_IND1:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[TMP5:%.*]] = add <4 x i16> [[VEC_IND1]], splat (i16 1)
+; THRESHOLD-NEXT: [[TMP6]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; THRESHOLD-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; THRESHOLD-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i64 0
+; THRESHOLD-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP8]]
+; THRESHOLD-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP9]], align 8
+; THRESHOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; THRESHOLD-NEXT: [[VEC_IND_NEXT2]] = add <4 x i16> [[VEC_IND1]], splat (i16 4)
+; THRESHOLD-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; THRESHOLD-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; THRESHOLD: [[MIDDLE_BLOCK]]:
+; THRESHOLD-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; THRESHOLD-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; THRESHOLD-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; THRESHOLD: [[SCALAR_PH]]:
+; THRESHOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i16 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD-NEXT: br label %[[LOOP1:.*]]
+; THRESHOLD: [[LOOP1]]:
+; THRESHOLD-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; THRESHOLD-NEXT: [[NARROW:%.*]] = phi i16 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP1]] ]
+; THRESHOLD-NEXT: [[PREV:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[EXT:%.*]], %[[LOOP1]] ]
; THRESHOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; THRESHOLD-NEXT: [[INC]] = add i16 [[NARROW]], 1
; THRESHOLD-NEXT: [[EXT]] = zext i16 [[INC]] to i64
; THRESHOLD-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[PREV]]
; THRESHOLD-NEXT: store i64 [[IV]], ptr [[GEP]], align 8
; THRESHOLD-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
; THRESHOLD: [[EXIT]]:
-; THRESHOLD-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
; THRESHOLD-NEXT: ret i64 [[RESULT]]
;
entry:
@@ -577,7 +614,7 @@ define void @for_and_ind_trunc_sole_canonical_iv(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
; THRESHOLD-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; THRESHOLD: [[MIDDLE_BLOCK]]:
; THRESHOLD-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP7]], i64 3
; THRESHOLD-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
@@ -598,7 +635,7 @@ define void @for_and_ind_trunc_sole_canonical_iv(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: store i32 [[PREV_TRUNC]], ptr [[GEP]], align 4
; THRESHOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; THRESHOLD-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
; THRESHOLD: [[EXIT]]:
; THRESHOLD-NEXT: ret void
;
@@ -698,7 +735,7 @@ define void @for_and_ind_trunc_another_canonical_iv(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
; THRESHOLD-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; THRESHOLD: [[MIDDLE_BLOCK]]:
; THRESHOLD-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP4]], i64 3
; THRESHOLD-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -719,7 +756,7 @@ define void @for_and_ind_trunc_another_canonical_iv(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: store i32 [[PREV_TRUNC]], ptr [[GEP]], align 4
; THRESHOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; THRESHOLD-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP12:![0-9]+]]
; THRESHOLD: [[EXIT]]:
; THRESHOLD-NEXT: ret void
;
@@ -840,7 +877,7 @@ define void @for_and_ind_trunc_predicate_not_implied(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 12)
; THRESHOLD-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; THRESHOLD: [[MIDDLE_BLOCK]]:
; THRESHOLD-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i64 3
; THRESHOLD-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -861,7 +898,7 @@ define void @for_and_ind_trunc_predicate_not_implied(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TRUNC]]
; THRESHOLD-NEXT: store i32 0, ptr [[GEP]], align 4
; THRESHOLD-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP12:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
; THRESHOLD: [[EXIT]]:
; THRESHOLD-NEXT: ret void
;
@@ -1156,7 +1193,7 @@ define void @for_and_ind_optsize_prev_first(ptr %dst, i32 %n) optsize {
; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
; THRESHOLD-NEXT: [[VEC_IND_NEXT8]] = add nuw <4 x i32> [[VEC_IND1]], splat (i32 4)
; THRESHOLD-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; THRESHOLD: [[MIDDLE_BLOCK]]:
; THRESHOLD-NEXT: br label %[[EXIT:.*]]
; THRESHOLD: [[EXIT]]:
@@ -1310,7 +1347,7 @@ define void @for_and_ind_optsize_prev_last(ptr %dst, i32 %n) optsize {
; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
; THRESHOLD-NEXT: [[VEC_IND_NEXT8]] = add nuw <4 x i32> [[VEC_IND1]], splat (i32 4)
; THRESHOLD-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; THRESHOLD: [[MIDDLE_BLOCK]]:
; THRESHOLD-NEXT: br label %[[EXIT:.*]]
; THRESHOLD: [[EXIT]]:
@@ -1397,11 +1434,48 @@ define i64 @for_and_ind_widest_type_modeled_as_induction(ptr noalias %dst, ptr n
; THRESHOLD-LABEL: define i64 @for_and_ind_widest_type_modeled_as_induction(
; THRESHOLD-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i32 [[N:%.*]]) {
; THRESHOLD-NEXT: [[ENTRY:.*]]:
+; THRESHOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 1)
+; THRESHOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4
+; THRESHOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; THRESHOLD: [[VECTOR_SCEVCHECK]]:
+; THRESHOLD-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 1)
+; THRESHOLD-NEXT: [[TMP1:%.*]] = add nsw i32 [[SMAX]], -1
+; THRESHOLD-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[TMP1]], 65535
+; THRESHOLD-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; THRESHOLD: [[VECTOR_PH]]:
+; THRESHOLD-NEXT: [[TMP3:%.*]] = and i32 [[TMP0]], 3
+; THRESHOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP3]]
+; THRESHOLD-NEXT: [[TMP4:%.*]] = trunc i32 [[N_VEC]] to i16
; THRESHOLD-NEXT: br label %[[LOOP:.*]]
; THRESHOLD: [[LOOP]]:
-; THRESHOLD-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; THRESHOLD-NEXT: [[NARROW:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[LOOP]] ]
-; THRESHOLD-NEXT: [[PREV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[EXT:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[TMP5:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 1)
+; THRESHOLD-NEXT: [[TMP6]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; THRESHOLD-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; THRESHOLD-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i64 0
+; THRESHOLD-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP8]]
+; THRESHOLD-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; THRESHOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[DST]], i32 [[INDEX]]
+; THRESHOLD-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP10]], align 8
+; THRESHOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; THRESHOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; THRESHOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; THRESHOLD: [[MIDDLE_BLOCK]]:
+; THRESHOLD-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; THRESHOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; THRESHOLD-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; THRESHOLD: [[SCALAR_PH]]:
+; THRESHOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i16 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD-NEXT: br label %[[LOOP1:.*]]
+; THRESHOLD: [[LOOP1]]:
+; THRESHOLD-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; THRESHOLD-NEXT: [[NARROW:%.*]] = phi i16 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP1]] ]
+; THRESHOLD-NEXT: [[PREV:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[EXT:%.*]], %[[LOOP1]] ]
; THRESHOLD-NEXT: [[INC]] = add i16 [[NARROW]], 1
; THRESHOLD-NEXT: [[EXT]] = zext i16 [[INC]] to i64
; THRESHOLD-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[PREV]]
@@ -1410,9 +1484,9 @@ define i64 @for_and_ind_widest_type_modeled_as_induction(ptr noalias %dst, ptr n
; THRESHOLD-NEXT: store i64 [[L]], ptr [[GEP]], align 8
; THRESHOLD-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; THRESHOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; THRESHOLD-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
; THRESHOLD: [[EXIT]]:
-; THRESHOLD-NEXT: [[RES:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ]
+; THRESHOLD-NEXT: [[RES:%.*]] = phi i64 [ [[EXT]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
; THRESHOLD-NEXT: ret i64 [[RES]]
;
entry:
@@ -1617,7 +1691,7 @@ define i64 @for_and_ind_dead_in_body_live_out(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; THRESHOLD-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 12)
; THRESHOLD-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; THRESHOLD: [[MIDDLE_BLOCK]]:
; THRESHOLD-NEXT: [[TMP5:%.*]] = add <4 x i8> [[VEC_IND]], splat (i8 3)
; THRESHOLD-NEXT: [[TMP6:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i64>
@@ -1640,7 +1714,7 @@ define i64 @for_and_ind_dead_in_body_live_out(ptr %dst, i64 %n) {
; THRESHOLD-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[J]]
; THRESHOLD-NEXT: store i8 2, ptr [[GEP]], align 1
; THRESHOLD-NEXT: [[DONE:%.*]] = icmp ult i64 [[J_NEXT]], [[N]]
-; THRESHOLD-NEXT: br i1 [[DONE]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; THRESHOLD-NEXT: br i1 [[DONE]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !llvm.loop [[LOOP20:![0-9]+]]
; THRESHOLD: [[EXIT_LOOPEXIT]]:
; THRESHOLD-NEXT: [[WIDE_LCSSA:%.*]] = phi i64 [ [[WIDE]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
; THRESHOLD-NEXT: br label %[[EXIT]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
index 80d32ca536ced..fd0edf692fd8b 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
@@ -68,20 +68,57 @@ define i64 @predicated_iv_with_liveout(ptr %dst, i64 %n) {
; THRESHOLD0-LABEL: define i64 @predicated_iv_with_liveout(
; THRESHOLD0-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; THRESHOLD0-NEXT: [[ENTRY:.*]]:
+; THRESHOLD0-NEXT: [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; THRESHOLD0-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; THRESHOLD0-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; THRESHOLD0: [[VECTOR_SCEVCHECK]]:
+; THRESHOLD0-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; THRESHOLD0-NEXT: [[TMP1:%.*]] = add nsw i64 [[SMAX]], -1
+; THRESHOLD0-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[TMP1]], 65535
+; THRESHOLD0-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; THRESHOLD0: [[VECTOR_PH]]:
+; THRESHOLD0-NEXT: [[TMP3:%.*]] = and i64 [[TMP0]], 3
+; THRESHOLD0-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP3]]
+; THRESHOLD0-NEXT: [[TMP4:%.*]] = trunc i64 [[N_VEC]] to i16
; THRESHOLD0-NEXT: br label %[[LOOP:.*]]
; THRESHOLD0: [[LOOP]]:
-; THRESHOLD0-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; THRESHOLD0-NEXT: [[DEAD_IV:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[DEAD_IV_NEXT:%.*]], %[[LOOP]] ]
-; THRESHOLD0-NEXT: [[PREV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[EXT:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[VEC_IND1:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[TMP5:%.*]] = add <4 x i16> [[VEC_IND1]], splat (i16 1)
+; THRESHOLD0-NEXT: [[TMP6]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; THRESHOLD0-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP6]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; THRESHOLD0-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i64 0
+; THRESHOLD0-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP8]]
+; THRESHOLD0-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP9]], align 8
+; THRESHOLD0-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; THRESHOLD0-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; THRESHOLD0-NEXT: [[VEC_IND_NEXT2]] = add <4 x i16> [[VEC_IND1]], splat (i16 4)
+; THRESHOLD0-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; THRESHOLD0-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; THRESHOLD0: [[MIDDLE_BLOCK]]:
+; THRESHOLD0-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; THRESHOLD0-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; THRESHOLD0-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; THRESHOLD0: [[SCALAR_PH]]:
+; THRESHOLD0-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD0-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i16 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD0-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; THRESHOLD0-NEXT: br label %[[LOOP1:.*]]
+; THRESHOLD0: [[LOOP1]]:
+; THRESHOLD0-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; THRESHOLD0-NEXT: [[DEAD_IV:%.*]] = phi i16 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[DEAD_IV_NEXT:%.*]], %[[LOOP1]] ]
+; THRESHOLD0-NEXT: [[PREV:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[EXT:%.*]], %[[LOOP1]] ]
; THRESHOLD0-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; THRESHOLD0-NEXT: [[DEAD_IV_NEXT]] = add i16 [[DEAD_IV]], 1
; THRESHOLD0-NEXT: [[EXT]] = zext i16 [[DEAD_IV_NEXT]] to i64
; THRESHOLD0-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[PREV]]
; THRESHOLD0-NEXT: store i64 [[IV]], ptr [[GEP]], align 8
; THRESHOLD0-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD0-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; THRESHOLD0-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; THRESHOLD0: [[EXIT]]:
-; THRESHOLD0-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP]] ]
+; THRESHOLD0-NEXT: [[RESULT:%.*]] = phi i64 [ [[EXT]], %[[LOOP1]] ], [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ]
; THRESHOLD0-NEXT: ret i64 [[RESULT]]
;
; THRESHOLD1-LABEL: define i64 @predicated_iv_with_liveout(
@@ -224,7 +261,7 @@ define void @dead_predicated_iv1(ptr %dst, i64 %n) {
; THRESHOLD0-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; THRESHOLD0-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
; THRESHOLD0-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; THRESHOLD0-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; THRESHOLD0-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; THRESHOLD0: [[MIDDLE_BLOCK]]:
; THRESHOLD0-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; THRESHOLD0-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -241,7 +278,7 @@ define void @dead_predicated_iv1(ptr %dst, i64 %n) {
; THRESHOLD0-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
; THRESHOLD0-NEXT: store i64 [[EXT]], ptr [[GEP]], align 8
; THRESHOLD0-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; THRESHOLD0-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; THRESHOLD0-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; THRESHOLD0: [[EXIT]]:
; THRESHOLD0-NEXT: ret void
;
>From e4d28a4b301a2b2fdf98c1bbabc7f8993c5c3af9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 16 Sep 2026 09:25:45 +0100
Subject: [PATCH 2/2] [LV] Add comment
---
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index df11571314236..2d447e2050fd6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -444,6 +444,10 @@ static bool storeToSameAddress(ScalarEvolution *SE, StoreInst *A,
SmallVector<const SCEVPredicate *>
LoopVectorizationLegality::collectUnitStridePredicates() const {
+ // Since LoopAccessAnalysis' collectStridedAccess collects only unit-strided
+ // accesses, and SymbolicStrides is a map from a Value to a SCEVUnknown, with
+ // the implicit information that the SCEVUnknown was speculated to unit, we
+ // reconstruct SCEVUnknown = unit predicates here.
ScalarEvolution &SE = *PSE.getSE();
SmallVector<const SCEVPredicate *> Predicates;
if (!AllowRuntimeSCEVChecks || !TheLoop->isInnermost())
More information about the llvm-commits
mailing list