[llvm] [LV] Use SCEV to compute final value of complex induction variables (PR #195059)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 03:23:08 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/195059
>From 0e468662a07adb53f6ed9667bb738e98065284dd Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 17 Jul 2026 01:56:02 -0700
Subject: [PATCH 1/8] fix auto gen warning
---
.../LoopVectorize/iv_outside_user.ll | 104 +++++++++---------
1 file changed, 52 insertions(+), 52 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 94078c8413f41..8ef8146663269 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -183,14 +183,14 @@ define ptr @both(ptr %p, i32 %k) {
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; VEC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; VEC: [[SCALAR_PH]]:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; VEC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
-; VEC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi ptr [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
+; VEC-NEXT: [[TMP8:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VEC-NEXT: [[TMP9:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
+; VEC-NEXT: [[TMP10:%.*]] = phi ptr [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
; VEC-NEXT: br label %[[FOR_BODY:.*]]
; VEC: [[FOR_BODY]]:
-; VEC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; VEC-NEXT: [[INC_LAG1:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
-; VEC-NEXT: [[INC_LAG2:%.*]] = phi ptr [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
+; VEC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[TMP8]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; VEC-NEXT: [[INC_LAG1:%.*]] = phi ptr [ [[TMP9]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
+; VEC-NEXT: [[INC_LAG2:%.*]] = phi ptr [ [[TMP10]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
; VEC-NEXT: [[TMP]] = getelementptr inbounds i32, ptr [[INC_LAG1]], i64 1
; VEC-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
; VEC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
@@ -228,14 +228,14 @@ define ptr @both(ptr %p, i32 %k) {
; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTERLEAVE: [[SCALAR_PH]]:
-; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[NEXT_GEP1]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
-; INTERLEAVE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi ptr [ [[NEXT_GEP]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
+; INTERLEAVE-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = phi ptr [ [[NEXT_GEP1]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
+; INTERLEAVE-NEXT: [[TMP11:%.*]] = phi ptr [ [[NEXT_GEP]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
; INTERLEAVE-NEXT: br label %[[FOR_BODY:.*]]
; INTERLEAVE: [[FOR_BODY]]:
-; INTERLEAVE-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; INTERLEAVE-NEXT: [[INC_LAG1:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
-; INTERLEAVE-NEXT: [[INC_LAG2:%.*]] = phi ptr [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
+; INTERLEAVE-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[TMP12]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; INTERLEAVE-NEXT: [[INC_LAG1:%.*]] = phi ptr [ [[TMP10]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
+; INTERLEAVE-NEXT: [[INC_LAG2:%.*]] = phi ptr [ [[TMP11]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
; INTERLEAVE-NEXT: [[TMP]] = getelementptr inbounds i32, ptr [[INC_LAG1]], i64 1
; INTERLEAVE-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
; INTERLEAVE-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
@@ -346,15 +346,15 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[CMP_N12:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC7]]
; CHECK-NEXT: br i1 [[CMP_N12]], label %[[BB3:.*]], label %[[SCALAR_PH2]]
; CHECK: [[SCALAR_PH2]]:
-; CHECK-NEXT: [[BC_RESUME_VAL12:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK10]] ], [ [[TMP04]], %[[BB1]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL11:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK10]] ], [ [[TMP04]], %[[BB1]] ]
; CHECK-NEXT: br label %[[BB2:.*]]
; CHECK: [[BB2]]:
-; CHECK-NEXT: [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL12]], %[[SCALAR_PH2]] ], [ [[TMP06:%.*]], %[[BB2]] ]
+; CHECK-NEXT: [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL11]], %[[SCALAR_PH2]] ], [ [[TMP06:%.*]], %[[BB2]] ]
; CHECK-NEXT: [[TMP06]] = add i32 [[TMP05]], -8
; CHECK-NEXT: [[TMP07:%.*]] = icmp sgt i32 [[TMP06]], 0
; CHECK-NEXT: br i1 [[TMP07]], label %[[BB2]], label %[[BB3]], {{!llvm.loop ![0-9]+}}
; CHECK: [[BB3]]:
-; CHECK-NEXT: [[TMP08:%.*]] = phi i32 [ [[TMP05]], %[[BB2]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK10]] ]
+; CHECK-NEXT: [[IV1_LCSSA:%.*]] = phi i32 [ [[TMP05]], %[[BB2]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK10]] ]
; CHECK-NEXT: [[TMP09:%.*]] = sub i32 [[TMP00]], 4
; CHECK-NEXT: [[TMP10:%.*]] = icmp slt i32 [[TMP09]], 1
; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 [[TMP09]]
@@ -362,9 +362,9 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15
; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0)
; CHECK-NEXT: [[TMP19:%.*]] = sub i32 [[TMP11]], [[SMIN]]
-; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP19]], -8
-; CHECK-NEXT: [[TMP11:%.*]] = lshr i32 [[TMP10]], 3
-; CHECK-NEXT: [[TMP14:%.*]] = add nuw nsw i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP21:%.*]] = add i32 [[TMP19]], -8
+; CHECK-NEXT: [[TMP20:%.*]] = lshr i32 [[TMP21]], 3
+; CHECK-NEXT: [[TMP14:%.*]] = add nuw nsw i32 [[TMP20]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP14]], 2
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -385,41 +385,41 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[BB3]] ]
; CHECK-NEXT: br label %[[BB4:.*]]
; CHECK: [[BB4]]:
-; CHECK-NEXT: [[TMP16:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IND_END:%.*]], %[[BB4]] ]
-; CHECK-NEXT: [[IND_END]] = add i32 [[TMP16]], -8
-; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[IND_END]], 0
-; CHECK-NEXT: br i1 [[TMP13]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
+; CHECK-NEXT: [[IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[BB4]] ]
+; CHECK-NEXT: [[IV2_NEXT]] = add i32 [[IV2]], -8
+; CHECK-NEXT: [[TMP25:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP25]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
;
-BB0:
- br label %BB1
-
-BB1:
- %tmp00 = load i32, ptr %p, align 16
- %tmp01 = sub i32 %tmp00, 3
- %tmp02 = icmp slt i32 %tmp01, 1
- %tmp03 = select i1 %tmp02, i32 1, i32 %tmp01
- %tmp04 = add nsw i32 %tmp03, -7
- br label %BB2
-
-BB2:
- %tmp05 = phi i32 [ %tmp04, %BB1 ], [ %tmp06, %BB2 ]
- %tmp06 = add i32 %tmp05, -8
- %tmp07 = icmp sgt i32 %tmp06, 0
- br i1 %tmp07, label %BB2, label %BB3
-
-BB3:
- %tmp08 = phi i32 [ %tmp05, %BB2 ]
- %tmp09 = sub i32 %tmp00, 4
- %tmp10 = icmp slt i32 %tmp09, 1
- %tmp11 = select i1 %tmp10, i32 1, i32 %tmp09
- %tmp11.inc = add nsw i32 %tmp11, -7
- br label %BB4
-
-BB4:
- %tmp13 = phi i32 [ %tmp11.inc, %BB3 ], [ %tmp14, %BB4 ]
- %tmp14 = add i32 %tmp13, -8
- %tmp15 = icmp sgt i32 %tmp14, 0
- br i1 %tmp15, label %BB4, label %BB1
+entry:
+ br label %outer
+
+outer:
+ %n = load i32, ptr %p, align 16
+ %0 = sub i32 %n, 3
+ %1 = icmp slt i32 %0, 1
+ %2 = select i1 %1, i32 1, i32 %0
+ %start1 = add nsw i32 %2, -7
+ br label %loop1
+
+loop1:
+ %iv1 = phi i32 [ %start1, %outer ], [ %iv1.next, %loop1 ]
+ %iv1.next = add i32 %iv1, -8
+ %3 = icmp sgt i32 %iv1.next, 0
+ br i1 %3, label %loop1, label %loop1.exit
+
+loop1.exit:
+ %iv1.lcssa = phi i32 [ %iv1, %loop1 ]
+ %4 = sub i32 %n, 4
+ %5 = icmp slt i32 %4, 1
+ %6 = select i1 %5, i32 1, i32 %4
+ %start2 = add nsw i32 %6, -7
+ br label %loop2
+
+loop2:
+ %iv2 = phi i32 [ %start2, %loop1.exit ], [ %iv2.next, %loop2 ]
+ %iv2.next = add i32 %iv2, -8
+ %7 = icmp sgt i32 %iv2.next, 0
+ br i1 %7, label %loop2, label %outer
}
;
>From 4f98125dbfde6c0d7f4e69e584b8868ce31ad96e Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 17 Jul 2026 00:37:03 -0700
Subject: [PATCH 2/8] live-out tail folding test
---
.../LoopVectorize/iv_outside_user.ll | 620 ++++++++++++++++++
1 file changed, 620 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 8ef8146663269..0228218710f4b 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+" --version 5
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=CHECK,VEC %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=CHECK,INTERLEAVE %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data < %s | FileCheck --check-prefixes=TAILFOLD %s
define i32 @postinc(i32 %k) {
; CHECK-LABEL: define i32 @postinc(
@@ -32,6 +33,25 @@ define i32 @postinc(i32 %k) {
; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[K]]
+;
entry:
br label %for.body
@@ -76,6 +96,26 @@ define i32 @preinc(i32 %k) {
; CHECK-NEXT: [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_PHI_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @preinc(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[K]], 1
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[IND_ESCAPE]]
+;
entry:
br label %for.body
@@ -105,6 +145,21 @@ define i32 @constpre() {
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret i32 2
;
+; TAILFOLD-LABEL: define i32 @constpre() {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 2
+;
entry:
br label %for.body
@@ -137,6 +192,24 @@ define ptr @geppre(ptr %ptr) {
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret ptr [[IND_ESCAPE]]
;
+; TAILFOLD-LABEL: define ptr @geppre(
+; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret ptr [[IND_ESCAPE]]
+;
entry:
br label %for.body
@@ -244,6 +317,32 @@ define ptr @both(ptr %p, i32 %k) {
; INTERLEAVE-NEXT: [[INC_LAG1_LCSSA:%.*]] = phi ptr [ [[INC_LAG1]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret ptr [[INC_LAG1_LCSSA]]
;
+; TAILFOLD-LABEL: define ptr @both(
+; TAILFOLD-SAME: ptr [[P:%.*]], i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[BASE:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 1
+; TAILFOLD-NEXT: [[TMP0:%.*]] = add i32 [[K]], -1
+; TAILFOLD-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; TAILFOLD-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; TAILFOLD-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP3]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP4]], i64 -4
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret ptr [[IND_ESCAPE]]
+;
entry:
%base = getelementptr inbounds i32, ptr %p, i64 1
br label %for.body
@@ -293,6 +392,26 @@ define i32 @multiphi(i32 %k, ptr %p) {
; CHECK-NEXT: store i32 [[PHI2]], ptr [[P]], align 4
; CHECK-NEXT: ret i32 [[PHI]]
;
+; TAILFOLD-LABEL: define i32 @multiphi(
+; TAILFOLD-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: store i32 [[K]], ptr [[P]], align 4
+; TAILFOLD-NEXT: ret i32 [[K]]
+;
entry:
br label %for.body
@@ -390,6 +509,65 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[TMP25:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP25]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
;
+; TAILFOLD-LABEL: define void @PR30742(
+; TAILFOLD-SAME: ptr [[P:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[OUTER:.*]]
+; TAILFOLD: [[OUTER_LOOPEXIT:.*]]:
+; TAILFOLD-NEXT: br label %[[OUTER]]
+; TAILFOLD: [[OUTER]]:
+; TAILFOLD-NEXT: [[N:%.*]] = load i32, ptr [[P]], align 16
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sub i32 [[N]], 3
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 1
+; TAILFOLD-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP0]]
+; TAILFOLD-NEXT: [[START1:%.*]] = add nsw i32 [[TMP2]], -7
+; TAILFOLD-NEXT: [[TMP3:%.*]] = add nsw i32 [[TMP2]], -15
+; TAILFOLD-NEXT: [[TMP4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP3]], i32 0)
+; TAILFOLD-NEXT: [[TMP5:%.*]] = sub i32 [[TMP2]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], -8
+; TAILFOLD-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP6]], 3
+; TAILFOLD-NEXT: [[TMP8:%.*]] = add nuw nsw i32 [[TMP7]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH1:.*]]
+; TAILFOLD: [[VECTOR_PH1]]:
+; TAILFOLD-NEXT: [[N_RND_UP2:%.*]] = add i32 [[TMP8]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF3:%.*]] = urem i32 [[N_RND_UP2]], 2
+; TAILFOLD-NEXT: [[N_VEC4:%.*]] = sub i32 [[N_RND_UP2]], [[N_MOD_VF3]]
+; TAILFOLD-NEXT: [[TMP9:%.*]] = mul i32 [[TMP8]], -8
+; TAILFOLD-NEXT: [[TMP10:%.*]] = add i32 [[START1]], [[TMP9]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY5:.*]]
+; TAILFOLD: [[VECTOR_BODY5]]:
+; TAILFOLD-NEXT: [[INDEX6:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT7:%.*]], %[[VECTOR_BODY5]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX6]], 2
+; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT7]], [[N_VEC4]]
+; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK8:.*]], label %[[VECTOR_BODY5]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK8]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[TMP10]], -8
+; TAILFOLD-NEXT: br label %[[LOOP1_EXIT:.*]]
+; TAILFOLD: [[LOOP1_EXIT]]:
+; TAILFOLD-NEXT: [[TMP12:%.*]] = sub i32 [[N]], 4
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP12]], 1
+; TAILFOLD-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 [[TMP12]]
+; TAILFOLD-NEXT: [[START2:%.*]] = add nsw i32 [[TMP14]], -7
+; TAILFOLD-NEXT: [[TMP15:%.*]] = add nsw i32 [[TMP14]], -15
+; TAILFOLD-NEXT: [[TMP16:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP15]], i32 0)
+; TAILFOLD-NEXT: [[TMP17:%.*]] = sub i32 [[TMP14]], [[TMP16]]
+; TAILFOLD-NEXT: [[TMP18:%.*]] = add i32 [[TMP17]], -8
+; TAILFOLD-NEXT: [[TMP19:%.*]] = lshr i32 [[TMP18]], 3
+; TAILFOLD-NEXT: [[TMP20:%.*]] = add nuw nsw i32 [[TMP19]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP20]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[OUTER_LOOPEXIT]]
+;
entry:
br label %outer
@@ -465,6 +643,26 @@ define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
; INTERLEAVE: [[EXIT]]:
; INTERLEAVE-NEXT: ret i64 1001
;
+; TAILFOLD-LABEL: define i64 @iv_scalar_steps_and_outside_users(
+; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i64 1001
+;
entry:
br label %loop
@@ -525,6 +723,26 @@ define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
; INTERLEAVE: [[EXIT]]:
; INTERLEAVE-NEXT: ret i32 2002
;
+; TAILFOLD-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
+; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i32 2002
+;
entry:
br label %loop
@@ -574,6 +792,26 @@ define i32 @postinc_sub(i32 %k) {
; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc_sub(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sub i32 [[K]], [[K]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[TMP0]]
+;
entry:
br label %for.body
@@ -617,6 +855,25 @@ define i32 @postinc_swapped_ops(i32 %k) {
; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc_swapped_ops(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[K]]
+;
entry:
br label %for.body
@@ -697,6 +954,33 @@ define i32 @postinc_not_iv_backedge_value(i32 %k) {
; INTERLEAVE-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[INC_2_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[K]])
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP1:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP2]], i1 false)
+; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[TMP3]]
+;
entry:
br label %for.body
@@ -809,6 +1093,49 @@ define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N,
; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]]
;
+; TAILFOLD-LABEL: define float @fp_postinc_use_fadd(
+; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
+; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
+; TAILFOLD-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
+; TAILFOLD-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
+; TAILFOLD-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store float [[TMP5]], ptr [[TMP7]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = fadd fast float [[TMP5]], [[TMP11]]
+; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret float [[TMP2]]
+;
entry:
br label %loop
@@ -924,6 +1251,49 @@ define float @fp_postinc_use_fadd_ops_swapped(float %init, ptr noalias nocapture
; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]]
;
+; TAILFOLD-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
+; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
+; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
+; TAILFOLD-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
+; TAILFOLD-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
+; TAILFOLD-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store float [[TMP5]], ptr [[TMP7]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = fadd fast float [[TMP5]], [[TMP11]]
+; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret float [[TMP2]]
+;
entry:
br label %loop
@@ -1039,6 +1409,49 @@ define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N,
; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]]
;
+; TAILFOLD-LABEL: define float @fp_postinc_use_fsub(
+; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
+; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
+; TAILFOLD-NEXT: [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
+; TAILFOLD-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
+; TAILFOLD-NEXT: [[TMP5:%.*]] = fsub fast float [[INIT]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store float [[TMP5]], ptr [[TMP7]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float -1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = fsub fast float [[TMP5]], [[TMP11]]
+; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret float [[TMP2]]
+;
entry:
br label %loop
@@ -1101,6 +1514,26 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
; INTERLEAVE: [[E_EXIT]]:
; INTERLEAVE-NEXT: ret i32 8
;
+; TAILFOLD-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
+; TAILFOLD-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[INDEX]]
+; TAILFOLD-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP0]], align 2
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[E_EXIT:.*]]
+; TAILFOLD: [[E_EXIT]]:
+; TAILFOLD-NEXT: ret i32 8
+;
entry:
%step.1 = sext i8 0 to i32
%step.2 = add nsw i32 %step.1, 1
@@ -1144,6 +1577,30 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
; CHECK: [[E_EXIT]]:
; CHECK-NEXT: ret i32 8
;
+; TAILFOLD-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
+; TAILFOLD-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = shl i32 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
+; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
+; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]]
+; TAILFOLD-NEXT: store i16 0, ptr [[TMP2]], align 2
+; TAILFOLD-NEXT: store i16 0, ptr [[TMP3]], align 2
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
+; TAILFOLD-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[E_EXIT:.*]]
+; TAILFOLD: [[E_EXIT]]:
+; TAILFOLD-NEXT: ret i32 8
+;
entry:
%step.1 = sext i8 0 to i32
%step.2 = add nsw i32 %step.1, 1
@@ -1234,6 +1691,40 @@ define i32 @iv_ext_used_outside( ptr %dst) {
; INTERLEAVE-NEXT: [[ADD:%.*]] = add i32 [[IV_1_EXT_LCSSA]], [[IV_2_LCSSA]]
; INTERLEAVE-NEXT: ret i32 [[IV_1_EXT_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @iv_ext_used_outside(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 129)
+; TAILFOLD-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
+; TAILFOLD-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP0]]
+; TAILFOLD-NEXT: store i32 0, ptr [[TMP2]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP4:%.*]] = add i16 [[TMP0]], 1
+; TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP4]]
+; TAILFOLD-NEXT: store i32 0, ptr [[TMP5]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 130
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: [[ADD:%.*]] = add i32 129, 128
+; TAILFOLD-NEXT: ret i32 129
+;
entry:
br label %loop
@@ -1289,6 +1780,22 @@ define i64 @test_iv_increment_incremented(ptr %dst) {
; INTERLEAVE: [[EXIT]]:
; INTERLEAVE-NEXT: ret i64 1
;
+; TAILFOLD-LABEL: define i64 @test_iv_increment_incremented(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3
+; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[TMP0]], i64 -1
+; TAILFOLD-NEXT: store <2 x i16> splat (i16 1), ptr [[TMP1]], align 2
+; TAILFOLD-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i64 1
+;
entry:
br label %loop
@@ -1399,6 +1906,55 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; INTERLEAVE-NEXT: [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[IV_TRUNC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @cast_incremented_iv_live_out(
+; TAILFOLD-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
+; TAILFOLD-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP1]])
+; TAILFOLD-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
+; TAILFOLD-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
+; TAILFOLD-NEXT: [[TMP5:%.*]] = add i8 [[TMP4]], 1
+; TAILFOLD-NEXT: store i8 [[TMP5]], ptr [[TMP3]], align 1
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP7]]
+; TAILFOLD-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = add i8 [[TMP9]], 1
+; TAILFOLD-NEXT: store i8 [[TMP10]], ptr [[TMP8]], align 1
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP12:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
+; TAILFOLD-NEXT: [[TMP13:%.*]] = trunc <2 x i64> [[TMP12]] to <2 x i32>
+; TAILFOLD-NEXT: [[TMP14:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP14]], i1 false)
+; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TAILFOLD-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i32 [[TMP15]]
+;
entry:
br label %loop
@@ -1506,6 +2062,70 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; INTERLEAVE-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[DERIVED_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @added_step(
+; TAILFOLD-SAME: i32 [[N:%.*]], i32 [[STEP_BASE:%.*]], ptr [[P:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[STEP:%.*]] = add i32 [[STEP_BASE]], 1
+; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; TAILFOLD-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; TAILFOLD: [[VECTOR_SCEVCHECK]]:
+; TAILFOLD-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; TAILFOLD-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; TAILFOLD-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP0]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0
+; TAILFOLD-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[TMP0]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; TAILFOLD-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
+; TAILFOLD-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP8]]
+; TAILFOLD-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
+; TAILFOLD-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP12]], i1 false)
+; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TAILFOLD-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP3]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[SCALAR_PH]]:
+; TAILFOLD-NEXT: br label %[[LOOP:.*]]
+; TAILFOLD: [[LOOP]]:
+; TAILFOLD-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TAILFOLD-NEXT: [[DERIVED:%.*]] = mul i32 [[IV]], [[STEP]]
+; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; TAILFOLD-NEXT: store i32 [[DERIVED]], ptr [[GEP]], align 4
+; TAILFOLD-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; TAILFOLD-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
+; TAILFOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; TAILFOLD-NEXT: ret i32 [[DERIVED_LCSSA]]
+;
entry:
%step = add i32 %step_base, 1
br label %loop
>From b5b74b902969b4c2330150cae1bae001ea5d008b Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 23 Mar 2026 20:49:03 -0700
Subject: [PATCH 3/8] [First candidate] Use SCEV to compute final value of
complex IV
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 45 ++++++-
.../Transforms/Vectorize/VPlanTransforms.h | 3 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 18 ++-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 +
.../LoopVectorize/AArch64/reduction-cost.ll | 30 ++---
.../LoopVectorize/VPlan/vplan-printing.ll | 30 +++--
.../LoopVectorize/X86/gep-use-outside-loop.ll | 2 +-
...6-sunk-instruction-used-outside-of-loop.ll | 15 +--
.../instruction-only-used-outside-of-loop.ll | 5 +-
.../LoopVectorize/iv_outside_user.ll | 118 +++++++-----------
.../LoopVectorize/no_outside_user.ll | 5 +-
.../LoopVectorize/predicated-inductions.ll | 10 +-
13 files changed, 145 insertions(+), 146 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1929f52ae7d95..80a7b2fce178f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6635,7 +6635,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
*Plan, *TLI))
return nullptr;
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
+ OrigLoop);
return Plan;
}
@@ -6804,7 +6805,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Optimize FindIV reductions to use sentinel-based approach when possible.
RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
*OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
+ OrigLoop);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1dc13889af4d6..4810f86b336ae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1153,8 +1153,43 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
return nullptr;
}
+static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
+ PredicatedScalarEvolution &PSE,
+ VPValue *ResumeTC,
+ const Loop *L) {
+ VPValue *Incoming;
+ if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming))))
+ return nullptr;
+
+ const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
+ const SCEV *Start, *Step;
+ if (!match(IncomingSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step),
+ m_SpecificLoop(L))))
+ return nullptr;
+
+ // TODO: Start value can be defined be a VPExpandSCEVRecipe after
+ // VPDerivedIVRecipe supports a general VPValue as the start value.
+ VPIRValue *StartIRV = vputils::getVPIRValueForSCEVExpr(Plan, Start);
+ if (!StartIRV)
+ return nullptr;
+
+ Type *StartTy = StartIRV->getType();
+ assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
+ InductionDescriptor::InductionKind Kind =
+ StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
+ : InductionDescriptor::IK_IntInduction;
+ VPValue *StepVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Step);
+ VPBuilder Builder(cast<VPInstruction>(Op));
+ Type *TCTy = ResumeTC->getScalarType();
+ VPValue *ExitCount = Builder.createOverflowingOp(
+ Instruction::Sub, {ResumeTC, Plan.getConstantInt(TCTy, 1)},
+ {/*HasNUW=*/true, /*HasNSW=*/false}, DebugLoc::getUnknown());
+ return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, ExitCount,
+ StepVPV);
+}
+
void VPlanTransforms::optimizeInductionLiveOutUsers(
- VPlan &Plan, PredicatedScalarEvolution &PSE) {
+ VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
// Compute end values for all inductions.
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *VectorPH = cast<VPBasicBlock>(VectorRegion->getSinglePredecessor());
@@ -1190,12 +1225,16 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
VPValue *Escape = nullptr;
- if (PredVPBB == MiddleVPBB)
+ if (PredVPBB == MiddleVPBB) {
Escape = optimizeLatchExitInductionUser(
Plan, ExitIRI->getOperand(Idx), EndValues, PSE);
- else
+ if (!Escape)
+ Escape = optimizeLatchExitIVUserViaSCEV(
+ Plan, ExitIRI->getOperand(Idx), PSE, ResumeTC, L);
+ } else {
Escape = optimizeEarlyExitInductionUser(
Plan, ExitIRI->getOperand(Idx), PSE);
+ }
if (Escape)
ExitIRI->setOperand(Idx, Escape);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85375625d34b5..2eb5c1fd5b34b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -416,7 +416,8 @@ struct VPlanTransforms {
/// IV values by feeding them precomputed end values instead, possibly taken
/// one step backwards.
static void optimizeInductionLiveOutUsers(VPlan &Plan,
- PredicatedScalarEvolution &PSE);
+ PredicatedScalarEvolution &PSE,
+ const Loop *L);
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 03e1ead89c169..4220148a9788b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -38,16 +38,24 @@ bool vputils::onlyScalarValuesUsed(const VPValue *Def) {
[Def](const VPUser *U) { return U->usesScalars(Def); });
}
-VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
+VPIRValue *vputils::getVPIRValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
if (auto *E = dyn_cast<SCEVConstant>(Expr))
return Plan.getOrAddLiveIn(E->getValue());
- // Skip SCEV expansion if Expr is a SCEVUnknown wrapping a non-instruction
- // value. Otherwise the value may be defined in a loop and using it directly
- // will break LCSSA form. The SCEV expansion takes care of preserving LCSSA
- // form.
+ // For a SCEVUnknown wrapping an instruction, return nullptr since the value
+ // may be defined in a loop and using it directly will break LCSSA form.
auto *U = dyn_cast<SCEVUnknown>(Expr);
if (U && !isa<Instruction>(U->getValue()))
return Plan.getOrAddLiveIn(U->getValue());
+ return nullptr;
+}
+
+VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
+ // If SCEV expression can be represented as a live-in VPIRValue, use it
+ // directly without expanding.
+ if (auto *IRV = getVPIRValueForSCEVExpr(Plan, Expr))
+ return IRV;
+ // Otherwise use VPExpandSCEVRecipe to expand expression, which preserves
+ // LCSSA form for values defined in a loop.
auto *Expanded = new VPExpandSCEVRecipe(Expr);
VPBasicBlock *EntryVPBB = Plan.getEntry();
auto Iter = EntryVPBB->getFirstNonPhi();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2980b704ec8da..8b584ed39c367 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -32,6 +32,10 @@ bool onlyFirstPartUsed(const VPValue *Def);
/// Returns true if only scalar values of \p Def are used by all users.
bool onlyScalarValuesUsed(const VPValue *Def);
+/// If \p Expr is a SCEVConstant or a SCEVUnknown wrapping a non-instruction
+/// value, return the corresponding VPIRValue live-in. Otherwise return nullptr.
+VPIRValue *getVPIRValueForSCEVExpr(VPlan &Plan, const SCEV *Expr);
+
/// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
/// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
/// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
index 230a9d7325bc5..7f21fdeb2b766 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
@@ -9,35 +9,27 @@ define i64 @reduction(i64 %arg) #0 {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
; CHECK: [[VECTOR_PH1]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ARG]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-NEXT: [[TMP5]] = or <4 x i32> [[VEC_PHI]], splat (i32 1)
; CHECK-NEXT: [[TMP1]] = or <4 x i32> [[VEC_PHI2]], splat (i32 1)
; CHECK-NEXT: [[TMP16]] = or <4 x i32> [[VEC_PHI4]], splat (i32 1)
; CHECK-NEXT: [[TMP17]] = or <4 x i32> [[VEC_PHI3]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 4)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP18:%.*]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 1)
-; CHECK-NEXT: [[TMP19:%.*]] = mul nsw <4 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[BIN_RDX1:%.*]] = or <4 x i32> [[TMP1]], [[TMP5]]
; CHECK-NEXT: [[BIN_RDX4:%.*]] = or <4 x i32> [[TMP16]], [[BIN_RDX1]]
; CHECK-NEXT: [[BIN_RDX5:%.*]] = or <4 x i32> [[TMP17]], [[BIN_RDX4]]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX5]])
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP19]], i64 3
+; CHECK-NEXT: [[TMP8:%.*]] = mul i64 95, [[ARG]]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[ARG]], [[TMP8]]
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
@@ -45,41 +37,33 @@ define i64 @reduction(i64 %arg) #0 {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i64> poison, i64 [[ARG]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT6]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT8]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT9]], <i64 0, i64 1, i64 2, i64 3>
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT13:%.*]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VEC_IND11:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT14:%.*]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[VEC_PHI12:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP10:%.*]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[TMP10]] = or <4 x i32> [[VEC_PHI12]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT14]] = add nsw <4 x i64> [[VEC_IND11]], splat (i64 4)
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT13]], 100
; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[SCALAR_PH]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i64> [[VEC_IND11]], splat (i64 1)
-; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <4 x i64> [[TMP12]], [[BROADCAST_SPLAT7]]
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP10]])
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP13]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 99, [[ARG]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[ARG]], [[TMP12]]
; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX15:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[BC_MERGE_RDX15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[BC_MERGE_RDX9]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[TMP2]] = or i32 [[VEC_PHI1]], 1
; CHECK-NEXT: [[TMP3]] = add nsw i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw i64 [[TMP3]], [[ARG]]
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[TMP3]], 100
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i64 [ [[TMP4]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i64 [ [[TMP4]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[BIN_RDX:%.*]] = phi i32 [ [[TMP2]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i32 [[BIN_RDX]] to i64
; CHECK-NEXT: [[RES:%.*]] = add i64 [[EXT]], [[MUL_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index 2e588784eaf00..54a8f4e068fee 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -479,26 +479,26 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %div = udiv i64 %y, 492802768830814060
-; CHECK-NEXT: IR %inc = add i64 %div, 1
-; CHECK-NEXT: EMIT vp<[[VP4:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
-; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %div = udiv i64 %y, 492802768830814060
+; CHECK-NEXT: IR %inc = add i64 %div, 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
+; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP3]]>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, vp<[[VP4]]>, vp<[[VP0]]> (truncated to i8)
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, vp<[[VP3]]>, vp<[[VP0]]> (truncated to i8)
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%iv>, ir<1>
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP8]]>
; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep>
@@ -509,7 +509,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP4]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<loop.exit>, scalar.ph
; CHECK-EMPTY:
@@ -569,7 +569,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
@@ -581,9 +580,8 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = extract-last-part ir<%add>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extract-last-lane vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = sub nuw vp<[[VP2]]>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<%off> + vp<[[VP7]]> * ir<1>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1000>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll b/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll
index 8fdd60f2dc1b1..c3ea2a3f4fd76 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll
@@ -82,10 +82,10 @@ define void @gep_use_outside_loop(ptr noalias %dst, ptr %src) {
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <4 x i16> [[WIDE_LOAD]], splat (i16 10)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> zeroinitializer, ptr align 2 [[TMP6]], <4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
index 3d31aaab1341c..29cd22c0944f1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
@@ -9,15 +9,13 @@ define ptr @test(ptr noalias %src, ptr noalias %dst) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <2 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
@@ -26,22 +24,25 @@ define ptr @test(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP13]]
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP11]], i64 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
; CHECK: [[PRED_LOAD_CONTINUE2]]:
; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP15]], <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[SRC]], i64 3996
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret ptr [[TMP2]]
+; CHECK-NEXT: ret ptr [[TMP16]]
;
entry:
br label %loop.header
diff --git a/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
index 1bec39fbae92f..7ffeb8b3c2908 100644
--- a/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
@@ -150,16 +150,13 @@ define i32 @optimizable_trunc_used_outside() {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[VEC_IND]], i64 3
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: exit:
-; CHECK-NEXT: ret i32 [[TMP1]]
+; CHECK-NEXT: ret i32 999
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 0228218710f4b..cf85fb5e625ad 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -888,71 +888,37 @@ for.end:
}
define i32 @postinc_not_iv_backedge_value(i32 %k) {
-; VEC-LABEL: define i32 @postinc_not_iv_backedge_value(
-; VEC-SAME: i32 [[K:%.*]]) {
-; VEC-NEXT: [[ENTRY:.*]]:
-; VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; VEC: [[VECTOR_PH]]:
-; VEC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
-; VEC-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
-; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; VEC-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
-; VEC-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; VEC-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; VEC: [[MIDDLE_BLOCK]]:
-; VEC-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
-; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP0]], i64 1
-; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; VEC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; VEC: [[SCALAR_PH]]:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; VEC-NEXT: br label %[[FOR_BODY:.*]]
-; VEC: [[FOR_BODY]]:
-; VEC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; VEC-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; VEC-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
-; VEC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; VEC-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; VEC: [[FOR_END]]:
-; VEC-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
-; VEC-NEXT: ret i32 [[INC_2_LCSSA]]
-;
-; INTERLEAVE-LABEL: define i32 @postinc_not_iv_backedge_value(
-; INTERLEAVE-SAME: i32 [[K:%.*]]) {
-; INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; INTERLEAVE: [[VECTOR_PH]]:
-; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
-; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
-; INTERLEAVE: [[VECTOR_BODY]]:
-; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; INTERLEAVE-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; INTERLEAVE: [[MIDDLE_BLOCK]]:
-; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 1
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
-; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; INTERLEAVE: [[SCALAR_PH]]:
-; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; INTERLEAVE-NEXT: br label %[[FOR_BODY:.*]]
-; INTERLEAVE: [[FOR_BODY]]:
-; INTERLEAVE-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; INTERLEAVE-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; INTERLEAVE-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
-; INTERLEAVE-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; INTERLEAVE-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; INTERLEAVE: [[FOR_END]]:
-; INTERLEAVE-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
-; INTERLEAVE-NEXT: ret i32 [[INC_2_LCSSA]]
+; CHECK-LABEL: define i32 @postinc_not_iv_backedge_value(
+; CHECK-SAME: i32 [[K:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = sub nuw i32 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 2, [[TMP1]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
+; CHECK-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[INC_2_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
@@ -1827,19 +1793,17 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 1
; VEC-NEXT: [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
; VEC-NEXT: store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; VEC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC: [[MIDDLE_BLOCK]]:
-; VEC-NEXT: [[TMP4:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
-; VEC-NEXT: [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i32>
-; VEC-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
+; VEC-NEXT: [[TMP4:%.*]] = sub nuw i64 [[N_VEC]], 1
+; VEC-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+; VEC-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
; VEC-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC: [[SCALAR_PH]]:
@@ -1885,8 +1849,9 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; INTERLEAVE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE: [[MIDDLE_BLOCK]]:
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], 1
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = sub nuw i64 [[N_VEC]], 1
; INTERLEAVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; INTERLEAVE-NEXT: [[TMP11:%.*]] = add i32 1, [[TMP10]]
; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE: [[SCALAR_PH]]:
@@ -1903,7 +1868,7 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; INTERLEAVE-NEXT: [[COND:%.*]] = icmp ult i32 [[IV_TRUNC]], [[N]]
; INTERLEAVE-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE: [[EXIT]]:
-; INTERLEAVE-NEXT: [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; INTERLEAVE-NEXT: [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[IV_TRUNC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @cast_incremented_iv_live_out(
@@ -2003,7 +1968,8 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; VEC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC: [[MIDDLE_BLOCK]]:
-; VEC-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP2]], i64 1
+; VEC-NEXT: [[TMP5:%.*]] = sub nuw i32 [[N_VEC]], 1
+; VEC-NEXT: [[TMP6:%.*]] = mul i32 [[TMP5]], [[STEP]]
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]]
; VEC-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC: [[SCALAR_PH]]:
@@ -2018,7 +1984,7 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; VEC-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
; VEC-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; VEC: [[EXIT]]:
-; VEC-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
+; VEC-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT: ret i32 [[DERIVED_LCSSA]]
;
; INTERLEAVE-LABEL: define i32 @added_step(
@@ -2045,6 +2011,8 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; INTERLEAVE-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE: [[MIDDLE_BLOCK]]:
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = sub nuw i32 [[N_VEC]], 1
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], [[STEP]]
; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE: [[SCALAR_PH]]:
@@ -2059,7 +2027,7 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; INTERLEAVE-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
; INTERLEAVE-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE: [[EXIT]]:
-; INTERLEAVE-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
+; INTERLEAVE-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[DERIVED_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @added_step(
diff --git a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
index 74193bb5322fa..a947cb6e4df91 100644
--- a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
@@ -736,15 +736,12 @@ define i32 @non_uniform_live_out() {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 7)
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 7
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[TMP1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
; CHECK-NEXT: store <2 x i8> [[TMP4]], ptr [[TMP2]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20000
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
index 2a76604a286ff..7cf8a7a3a0829 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-inductions.ll
@@ -605,18 +605,18 @@ define i32 @predicated_iv_inc_liveout_with_lai_preds(ptr %dst, ptr %src, i64 %st
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 9, i32 18, i32 27>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 36)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP11:%.*]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 9)
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP11]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = sub nuw i64 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = mul i32 [[TMP16]], 9
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 9, [[TMP14]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -637,7 +637,7 @@ define i32 @predicated_iv_inc_liveout_with_lai_preds(ptr %dst, ptr %src, i64 %st
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_NEXT]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[PRED_NEXT]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RESULT]]
;
; THRESHOLD0-LABEL: define i32 @predicated_iv_inc_liveout_with_lai_preds(
>From dc9a81242a0724dfc7b542e2fbd1bd26f2d50183 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 10 Jul 2026 02:16:08 -0700
Subject: [PATCH 4/8] update comment and use VPSCEVExpander
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4810f86b336ae..d400a2f060536 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1163,23 +1163,30 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
const SCEV *Start, *Step;
- if (!match(IncomingSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step),
- m_SpecificLoop(L))))
+ if (!match(IncomingSCEV, m_scev_AffineAddRec(
+ m_Isa<SCEVConstant, SCEVUnknown>(m_SCEV(Start)),
+ m_SCEV(Step), m_SpecificLoop(L))))
return nullptr;
- // TODO: Start value can be defined be a VPExpandSCEVRecipe after
- // VPDerivedIVRecipe supports a general VPValue as the start value.
+ // TODO: Use VPSCEVExpander to expand Start once VPDerivedIVRecipe supports a
+ // general VPValue as the start value.
VPIRValue *StartIRV = vputils::getVPIRValueForSCEVExpr(Plan, Start);
if (!StartIRV)
return nullptr;
+ auto *ExtractR = cast<VPInstruction>(Op);
+ DebugLoc DL = ExtractR->getDebugLoc();
+ VPBuilder Builder(ExtractR);
+ VPValue *StepVPV =
+ VPSCEVExpander(Builder, *PSE.getSE(), DL).tryToExpand(Step);
+ if (!StepVPV)
+ return nullptr;
+
Type *StartTy = StartIRV->getType();
assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
InductionDescriptor::InductionKind Kind =
StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
: InductionDescriptor::IK_IntInduction;
- VPValue *StepVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Step);
- VPBuilder Builder(cast<VPInstruction>(Op));
Type *TCTy = ResumeTC->getScalarType();
VPValue *ExitCount = Builder.createOverflowingOp(
Instruction::Sub, {ResumeTC, Plan.getConstantInt(TCTy, 1)},
>From 73d39389f40b5ec8336dd3cded1d889d4c4283c1 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 10 Jul 2026 02:48:55 -0700
Subject: [PATCH 5/8] back to zext
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 ++
.../LoopVectorize/iv_outside_user.ll | 31 +++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d400a2f060536..1111da0bb8870 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1191,6 +1191,8 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
VPValue *ExitCount = Builder.createOverflowingOp(
Instruction::Sub, {ResumeTC, Plan.getConstantInt(TCTy, 1)},
{/*HasNUW=*/true, /*HasNSW=*/false}, DebugLoc::getUnknown());
+ ExitCount = Builder.createScalarZExtOrTrunc(
+ ExitCount, StepVPV->getScalarType(), TCTy, DebugLoc::getUnknown());
return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, ExitCount,
StepVPV);
}
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index cf85fb5e625ad..b633e4b1ad424 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -2110,3 +2110,34 @@ loop:
exit:
ret i32 %derived
}
+
+define i64 @zext_iv_outside_user() {
+; CHECK-LABEL: define i64 @zext_iv_outside_user() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 2147483651
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %wide = zext i32 %i to i64
+ %i.next = add i32 %i, 1
+ %c = icmp ult i32 %i.next, 2147483652
+ br i1 %c, label %loop, label %exit
+
+exit:
+ %r = phi i64 [ %wide, %loop ]
+ ret i64 %r
+}
>From 61f786fc5547c9e29bb0eb5bc8ad242ccf598bda Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 13 Jul 2026 01:49:39 -0700
Subject: [PATCH 6/8] Use zext instead of sext for Index in VPDerivedIVRecipe
expansion. nfc
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 +---
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c62c4a082b95e..3259fc9fc4d39 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3009,7 +3009,7 @@ InstructionCost VPDerivedIVRecipe::computeCost(ElementCount VF,
unsigned IndexTySize = IndexTy->getScalarSizeInBits();
if ((NeedsAdd || NeedsMul || NeedsShl) && StepTySize != IndexTySize) {
unsigned CastOpc =
- StepTySize < IndexTySize ? Instruction::Trunc : Instruction::SExt;
+ StepTySize < IndexTySize ? Instruction::Trunc : Instruction::ZExt;
Cost += Ctx.TTI.getCastInstrCost(
CastOpc, StepTy, IndexTy, TTI::CastContextHint::None, Ctx.CostKind);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1111da0bb8870..01fcd557d7712 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1191,8 +1191,6 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
VPValue *ExitCount = Builder.createOverflowingOp(
Instruction::Sub, {ResumeTC, Plan.getConstantInt(TCTy, 1)},
{/*HasNUW=*/true, /*HasNSW=*/false}, DebugLoc::getUnknown());
- ExitCount = Builder.createScalarZExtOrTrunc(
- ExitCount, StepVPV->getScalarType(), TCTy, DebugLoc::getUnknown());
return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, ExitCount,
StepVPV);
}
@@ -4166,7 +4164,7 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
Type *StepTy = Step->getScalarType();
Type *IndexTy = Index->getScalarType();
Index = StepTy->isIntegerTy()
- ? Builder.createScalarSExtOrTrunc(
+ ? Builder.createScalarZExtOrTrunc(
Index, StepTy, IndexTy, DebugLoc::getCompilerGenerated())
: Builder.createScalarCast(Instruction::SIToFP, Index, StepTy,
DebugLoc::getCompilerGenerated());
>From 05612f2324e0bc5282f47e80a7c17a5997f34a8f Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 13 Jul 2026 02:15:45 -0700
Subject: [PATCH 7/8] support the start need expand
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 22 +++++++------------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 18 +++++----------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 ----
.../AArch64/scalable-avoid-scalarization.ll | 11 ++++++----
4 files changed, 20 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 01fcd557d7712..00e190ee92675 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1163,26 +1163,20 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
const SCEV *Start, *Step;
- if (!match(IncomingSCEV, m_scev_AffineAddRec(
- m_Isa<SCEVConstant, SCEVUnknown>(m_SCEV(Start)),
- m_SCEV(Step), m_SpecificLoop(L))))
- return nullptr;
-
- // TODO: Use VPSCEVExpander to expand Start once VPDerivedIVRecipe supports a
- // general VPValue as the start value.
- VPIRValue *StartIRV = vputils::getVPIRValueForSCEVExpr(Plan, Start);
- if (!StartIRV)
+ if (!match(IncomingSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step),
+ m_SpecificLoop(L))))
return nullptr;
auto *ExtractR = cast<VPInstruction>(Op);
DebugLoc DL = ExtractR->getDebugLoc();
VPBuilder Builder(ExtractR);
- VPValue *StepVPV =
- VPSCEVExpander(Builder, *PSE.getSE(), DL).tryToExpand(Step);
- if (!StepVPV)
+ VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
+ VPValue *StartVPV = Expander.tryToExpand(Start);
+ VPValue *StepVPV = Expander.tryToExpand(Step);
+ if (!StartVPV || !StepVPV)
return nullptr;
- Type *StartTy = StartIRV->getType();
+ Type *StartTy = StartVPV->getScalarType();
assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
InductionDescriptor::InductionKind Kind =
StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
@@ -1191,7 +1185,7 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
VPValue *ExitCount = Builder.createOverflowingOp(
Instruction::Sub, {ResumeTC, Plan.getConstantInt(TCTy, 1)},
{/*HasNUW=*/true, /*HasNSW=*/false}, DebugLoc::getUnknown());
- return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, ExitCount,
+ return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartVPV, ExitCount,
StepVPV);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 4220148a9788b..03e1ead89c169 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -38,24 +38,16 @@ bool vputils::onlyScalarValuesUsed(const VPValue *Def) {
[Def](const VPUser *U) { return U->usesScalars(Def); });
}
-VPIRValue *vputils::getVPIRValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
+VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
if (auto *E = dyn_cast<SCEVConstant>(Expr))
return Plan.getOrAddLiveIn(E->getValue());
- // For a SCEVUnknown wrapping an instruction, return nullptr since the value
- // may be defined in a loop and using it directly will break LCSSA form.
+ // Skip SCEV expansion if Expr is a SCEVUnknown wrapping a non-instruction
+ // value. Otherwise the value may be defined in a loop and using it directly
+ // will break LCSSA form. The SCEV expansion takes care of preserving LCSSA
+ // form.
auto *U = dyn_cast<SCEVUnknown>(Expr);
if (U && !isa<Instruction>(U->getValue()))
return Plan.getOrAddLiveIn(U->getValue());
- return nullptr;
-}
-
-VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
- // If SCEV expression can be represented as a live-in VPIRValue, use it
- // directly without expanding.
- if (auto *IRV = getVPIRValueForSCEVExpr(Plan, Expr))
- return IRV;
- // Otherwise use VPExpandSCEVRecipe to expand expression, which preserves
- // LCSSA form for values defined in a loop.
auto *Expanded = new VPExpandSCEVRecipe(Expr);
VPBasicBlock *EntryVPBB = Plan.getEntry();
auto Iter = EntryVPBB->getFirstNonPhi();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 8b584ed39c367..2980b704ec8da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -32,10 +32,6 @@ bool onlyFirstPartUsed(const VPValue *Def);
/// Returns true if only scalar values of \p Def are used by all users.
bool onlyScalarValuesUsed(const VPValue *Def);
-/// If \p Expr is a SCEVConstant or a SCEVUnknown wrapping a non-instruction
-/// value, return the corresponding VPIRValue live-in. Otherwise return nullptr.
-VPIRValue *getVPIRValueForSCEVExpr(VPlan &Plan, const SCEV *Expr);
-
/// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
/// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
/// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
index 856afaaa5c5fb..9e9274b17148d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
@@ -44,10 +44,13 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 2
-; CHECK-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <vscale x 2 x ptr> [[TMP15]], i32 [[TMP14]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[IDX]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = shl nsw i64 [[TMP11]], 3
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = sub nuw i32 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT: [[TMP19:%.*]] = shl i64 [[TMP17]], 3
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP19]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[L_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
>From 6e42c286b727d4d0ff1a3bce2ee13b90cd98d8e7 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 17 Jul 2026 03:22:44 -0700
Subject: [PATCH 8/8] support live-out optimization when tail folding
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +++-
.../VPlan/buildvector-first-lane-only.ll | 13 +-----
.../LoopVectorize/iv_outside_user.ll | 42 ++++++++++---------
3 files changed, 31 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 00e190ee92675..c3b8fb4217bb5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1158,8 +1158,13 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
VPValue *ResumeTC,
const Loop *L) {
VPValue *Incoming;
- if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming))))
- return nullptr;
+ if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming)))) {
+ VPValue *Mask;
+ if (!match(Op, m_ExtractLane(m_LastActiveLane(m_VPValue(Mask)),
+ m_VPValue(Incoming))) ||
+ !match(Mask, m_HeaderMask()))
+ return nullptr;
+ }
const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
const SCEV *Start, *Step;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
index ded3bc66016fe..6d15c43baac10 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
@@ -4,31 +4,22 @@
define i16 @last_active_lane_live_out(i32 %x) {
; CHECK-LABEL: VPlan for loop in 'last_active_lane_live_out'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
-; CHECK-NEXT: Live-in ir<2> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = step-vector i16
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul vp<[[VP3]]>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>, ir<1>
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not vp<%active.lane.mask>
-; CHECK-NEXT: EMIT vp<%first.inactive.lane> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%last.active.lane> = sub vp<%first.inactive.lane>, ir<1>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = extractelement vp<[[VP5]]>, vp<%last.active.lane>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP7]]> from middle.block)
+; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP2]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index b633e4b1ad424..f8ee076c7b6db 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -931,18 +931,12 @@ define i32 @postinc_not_iv_backedge_value(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[K]])
; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
-; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP1:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
-; TAILFOLD-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP2]], i1 false)
-; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: [[TMP1:%.*]] = sub nuw i32 [[K]], 1
+; TAILFOLD-NEXT: [[TMP3:%.*]] = add i32 2, [[TMP1]]
; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
; TAILFOLD: [[FOR_END]]:
; TAILFOLD-NEXT: ret i32 [[TMP3]]
@@ -1884,7 +1878,6 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP1]])
; TAILFOLD-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
@@ -1906,16 +1899,12 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP12:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
-; TAILFOLD-NEXT: [[TMP13:%.*]] = trunc <2 x i64> [[TMP12]] to <2 x i32>
-; TAILFOLD-NEXT: [[TMP14:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP14]], i1 false)
-; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; TAILFOLD-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = sub nuw i64 [[TMP1]], 1
+; TAILFOLD-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i32
+; TAILFOLD-NEXT: [[TMP15:%.*]] = add i32 1, [[TMP13]]
; TAILFOLD-NEXT: br label %[[EXIT:.*]]
; TAILFOLD: [[EXIT]]:
; TAILFOLD-NEXT: ret i32 [[TMP15]]
@@ -2075,10 +2064,8 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP12]], i1 false)
-; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; TAILFOLD-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP3]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = sub nuw i32 [[TMP0]], 1
+; TAILFOLD-NEXT: [[TMP13:%.*]] = mul i32 [[TMP12]], [[STEP]]
; TAILFOLD-NEXT: br label %[[EXIT:.*]]
; TAILFOLD: [[SCALAR_PH]]:
; TAILFOLD-NEXT: br label %[[LOOP:.*]]
@@ -2127,6 +2114,21 @@ define i64 @zext_iv_outside_user() {
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 2147483651
;
+; TAILFOLD-LABEL: define i64 @zext_iv_outside_user() {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i64 2147483651
+;
entry:
br label %loop
More information about the llvm-commits
mailing list