[llvm] [LV] Enable the exit value optimization via SCEV for tail folding. (PR #210928)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 1 05:53:59 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/210928
>From 2900eb5965420b103db37117d4f90352d4c03698 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 17 Jul 2026 00:37:03 -0700
Subject: [PATCH 1/8] live-out tail folding test
---
.../LoopVectorize/iv_outside_user.ll | 620 ++++++++++++++++++
1 file changed, 620 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 8448f3eaba7d1..f2d91f8ebd607 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+" --version 5
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=CHECK,VEC %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=CHECK,INTERLEAVE %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data < %s | FileCheck --check-prefixes=TAILFOLD %s
define i32 @postinc(i32 %k) {
; CHECK-LABEL: define i32 @postinc(
@@ -32,6 +33,25 @@ define i32 @postinc(i32 %k) {
; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[K]]
+;
entry:
br label %for.body
@@ -76,6 +96,26 @@ define i32 @preinc(i32 %k) {
; CHECK-NEXT: [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_PHI_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @preinc(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[K]], 1
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[IND_ESCAPE]]
+;
entry:
br label %for.body
@@ -105,6 +145,21 @@ define i32 @constpre() {
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret i32 2
;
+; TAILFOLD-LABEL: define i32 @constpre() {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 2
+;
entry:
br label %for.body
@@ -137,6 +192,24 @@ define ptr @geppre(ptr %ptr) {
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret ptr [[IND_ESCAPE]]
;
+; TAILFOLD-LABEL: define ptr @geppre(
+; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret ptr [[IND_ESCAPE]]
+;
entry:
br label %for.body
@@ -244,6 +317,32 @@ define ptr @both(ptr %p, i32 %k) {
; INTERLEAVE-NEXT: [[INC_LAG1_LCSSA:%.*]] = phi ptr [ [[INC_LAG1]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret ptr [[INC_LAG1_LCSSA]]
;
+; TAILFOLD-LABEL: define ptr @both(
+; TAILFOLD-SAME: ptr [[P:%.*]], i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[BASE:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 1
+; TAILFOLD-NEXT: [[TMP0:%.*]] = add i32 [[K]], -1
+; TAILFOLD-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; TAILFOLD-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; TAILFOLD-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP3]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP4]], i64 -4
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret ptr [[IND_ESCAPE]]
+;
entry:
%base = getelementptr inbounds i32, ptr %p, i64 1
br label %for.body
@@ -293,6 +392,26 @@ define i32 @multiphi(i32 %k, ptr %p) {
; CHECK-NEXT: store i32 [[PHI2]], ptr [[P]], align 4
; CHECK-NEXT: ret i32 [[PHI]]
;
+; TAILFOLD-LABEL: define i32 @multiphi(
+; TAILFOLD-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: store i32 [[K]], ptr [[P]], align 4
+; TAILFOLD-NEXT: ret i32 [[K]]
+;
entry:
br label %for.body
@@ -390,6 +509,65 @@ define void @PR30742(ptr %p) {
; CHECK-NEXT: [[LOOP2_EC:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
; CHECK-NEXT: br i1 [[LOOP2_EC]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
;
+; TAILFOLD-LABEL: define void @PR30742(
+; TAILFOLD-SAME: ptr [[P:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[OUTER:.*]]
+; TAILFOLD: [[OUTER_LOOPEXIT:.*]]:
+; TAILFOLD-NEXT: br label %[[OUTER]]
+; TAILFOLD: [[OUTER]]:
+; TAILFOLD-NEXT: [[N:%.*]] = load i32, ptr [[P]], align 16
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sub i32 [[N]], 3
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 1
+; TAILFOLD-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP0]]
+; TAILFOLD-NEXT: [[START1:%.*]] = add nsw i32 [[TMP2]], -7
+; TAILFOLD-NEXT: [[TMP3:%.*]] = add nsw i32 [[TMP2]], -15
+; TAILFOLD-NEXT: [[TMP4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP3]], i32 0)
+; TAILFOLD-NEXT: [[TMP5:%.*]] = sub i32 [[TMP2]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], -8
+; TAILFOLD-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP6]], 3
+; TAILFOLD-NEXT: [[TMP8:%.*]] = add nuw nsw i32 [[TMP7]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH1:.*]]
+; TAILFOLD: [[VECTOR_PH1]]:
+; TAILFOLD-NEXT: [[N_RND_UP2:%.*]] = add i32 [[TMP8]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF3:%.*]] = urem i32 [[N_RND_UP2]], 2
+; TAILFOLD-NEXT: [[N_VEC4:%.*]] = sub i32 [[N_RND_UP2]], [[N_MOD_VF3]]
+; TAILFOLD-NEXT: [[TMP9:%.*]] = mul i32 [[TMP8]], -8
+; TAILFOLD-NEXT: [[TMP10:%.*]] = add i32 [[START1]], [[TMP9]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY5:.*]]
+; TAILFOLD: [[VECTOR_BODY5]]:
+; TAILFOLD-NEXT: [[INDEX6:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT7:%.*]], %[[VECTOR_BODY5]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX6]], 2
+; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT7]], [[N_VEC4]]
+; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK8:.*]], label %[[VECTOR_BODY5]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK8]]:
+; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[TMP10]], -8
+; TAILFOLD-NEXT: br label %[[LOOP1_EXIT:.*]]
+; TAILFOLD: [[LOOP1_EXIT]]:
+; TAILFOLD-NEXT: [[TMP12:%.*]] = sub i32 [[N]], 4
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP12]], 1
+; TAILFOLD-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 [[TMP12]]
+; TAILFOLD-NEXT: [[START2:%.*]] = add nsw i32 [[TMP14]], -7
+; TAILFOLD-NEXT: [[TMP15:%.*]] = add nsw i32 [[TMP14]], -15
+; TAILFOLD-NEXT: [[TMP16:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP15]], i32 0)
+; TAILFOLD-NEXT: [[TMP17:%.*]] = sub i32 [[TMP14]], [[TMP16]]
+; TAILFOLD-NEXT: [[TMP18:%.*]] = add i32 [[TMP17]], -8
+; TAILFOLD-NEXT: [[TMP19:%.*]] = lshr i32 [[TMP18]], 3
+; TAILFOLD-NEXT: [[TMP20:%.*]] = add nuw nsw i32 [[TMP19]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP20]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[OUTER_LOOPEXIT]]
+;
entry:
br label %outer
@@ -465,6 +643,26 @@ define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
; INTERLEAVE: [[EXIT]]:
; INTERLEAVE-NEXT: ret i64 1001
;
+; TAILFOLD-LABEL: define i64 @iv_scalar_steps_and_outside_users(
+; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i64 1001
+;
entry:
br label %loop
@@ -525,6 +723,26 @@ define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
; INTERLEAVE: [[EXIT]]:
; INTERLEAVE-NEXT: ret i32 2002
;
+; TAILFOLD-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
+; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i32 2002
+;
entry:
br label %loop
@@ -574,6 +792,26 @@ define i32 @postinc_sub(i32 %k) {
; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc_sub(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sub i32 [[K]], [[K]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[TMP0]]
+;
entry:
br label %for.body
@@ -617,6 +855,25 @@ define i32 @postinc_swapped_ops(i32 %k) {
; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc_swapped_ops(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[K]]
+;
entry:
br label %for.body
@@ -663,6 +920,33 @@ define i32 @postinc_not_iv_backedge_value(i32 %k) {
; CHECK-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[INC_2_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
+; TAILFOLD-SAME: i32 [[K:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[K]])
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP1:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP2]], i1 false)
+; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
+; TAILFOLD: [[FOR_END]]:
+; TAILFOLD-NEXT: ret i32 [[TMP3]]
+;
entry:
br label %for.body
@@ -810,6 +1094,49 @@ define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N,
; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]]
;
+; TAILFOLD-LABEL: define float @fp_postinc_use_fadd(
+; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
+; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
+; TAILFOLD-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
+; TAILFOLD-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
+; TAILFOLD-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store float [[TMP5]], ptr [[TMP7]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = fadd fast float [[TMP5]], [[TMP11]]
+; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret float [[TMP2]]
+;
entry:
br label %loop
@@ -925,6 +1252,49 @@ define float @fp_postinc_use_fadd_ops_swapped(float %init, ptr noalias nocapture
; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]]
;
+; TAILFOLD-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
+; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
+; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
+; TAILFOLD-NEXT: [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
+; TAILFOLD-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
+; TAILFOLD-NEXT: [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store float [[TMP5]], ptr [[TMP7]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = fadd fast float [[TMP5]], [[TMP11]]
+; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret float [[TMP2]]
+;
entry:
br label %loop
@@ -1040,6 +1410,49 @@ define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N,
; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]]
;
+; TAILFOLD-LABEL: define float @fp_postinc_use_fsub(
+; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
+; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
+; TAILFOLD-NEXT: [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
+; TAILFOLD-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
+; TAILFOLD-NEXT: [[TMP5:%.*]] = fsub fast float [[INIT]], [[TMP4]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; TAILFOLD-NEXT: store float [[TMP5]], ptr [[TMP7]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float -1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = fsub fast float [[TMP5]], [[TMP11]]
+; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret float [[TMP2]]
+;
entry:
br label %loop
@@ -1102,6 +1515,26 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
; INTERLEAVE: [[E_EXIT]]:
; INTERLEAVE-NEXT: ret i32 8
;
+; TAILFOLD-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
+; TAILFOLD-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[INDEX]]
+; TAILFOLD-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP0]], align 2
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[E_EXIT:.*]]
+; TAILFOLD: [[E_EXIT]]:
+; TAILFOLD-NEXT: ret i32 8
+;
entry:
%step.1 = sext i8 0 to i32
%step.2 = add nsw i32 %step.1, 1
@@ -1145,6 +1578,30 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
; CHECK: [[E_EXIT]]:
; CHECK-NEXT: ret i32 8
;
+; TAILFOLD-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
+; TAILFOLD-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = shl i32 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
+; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
+; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]]
+; TAILFOLD-NEXT: store i16 0, ptr [[TMP2]], align 2
+; TAILFOLD-NEXT: store i16 0, ptr [[TMP3]], align 2
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
+; TAILFOLD-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[E_EXIT:.*]]
+; TAILFOLD: [[E_EXIT]]:
+; TAILFOLD-NEXT: ret i32 8
+;
entry:
%step.1 = sext i8 0 to i32
%step.2 = add nsw i32 %step.1, 1
@@ -1235,6 +1692,40 @@ define i32 @iv_ext_used_outside( ptr %dst) {
; INTERLEAVE-NEXT: [[ADD:%.*]] = add i32 [[IV_1_EXT_LCSSA]], [[IV_2_LCSSA]]
; INTERLEAVE-NEXT: ret i32 [[IV_1_EXT_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @iv_ext_used_outside(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 129)
+; TAILFOLD-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
+; TAILFOLD-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP0]]
+; TAILFOLD-NEXT: store i32 0, ptr [[TMP2]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP4:%.*]] = add i16 [[TMP0]], 1
+; TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP4]]
+; TAILFOLD-NEXT: store i32 0, ptr [[TMP5]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 130
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: [[ADD:%.*]] = add i32 129, 128
+; TAILFOLD-NEXT: ret i32 129
+;
entry:
br label %loop
@@ -1290,6 +1781,22 @@ define i64 @test_iv_increment_incremented(ptr %dst) {
; INTERLEAVE: [[EXIT]]:
; INTERLEAVE-NEXT: ret i64 1
;
+; TAILFOLD-LABEL: define i64 @test_iv_increment_incremented(
+; TAILFOLD-SAME: ptr [[DST:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3
+; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[TMP0]], i64 -1
+; TAILFOLD-NEXT: store <2 x i16> splat (i16 1), ptr [[TMP1]], align 2
+; TAILFOLD-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i64 1
+;
entry:
br label %loop
@@ -1399,6 +1906,55 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; INTERLEAVE-NEXT: [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[IV_TRUNC_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @cast_incremented_iv_live_out(
+; TAILFOLD-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
+; TAILFOLD-NEXT: [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP1]])
+; TAILFOLD-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
+; TAILFOLD-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
+; TAILFOLD-NEXT: [[TMP5:%.*]] = add i8 [[TMP4]], 1
+; TAILFOLD-NEXT: store i8 [[TMP5]], ptr [[TMP3]], align 1
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP7]]
+; TAILFOLD-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1
+; TAILFOLD-NEXT: [[TMP10:%.*]] = add i8 [[TMP9]], 1
+; TAILFOLD-NEXT: store i8 [[TMP10]], ptr [[TMP8]], align 1
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP12:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
+; TAILFOLD-NEXT: [[TMP13:%.*]] = trunc <2 x i64> [[TMP12]] to <2 x i32>
+; TAILFOLD-NEXT: [[TMP14:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP14]], i1 false)
+; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TAILFOLD-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i32 [[TMP15]]
+;
entry:
br label %loop
@@ -1508,6 +2064,70 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; INTERLEAVE-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[DERIVED_LCSSA]]
;
+; TAILFOLD-LABEL: define i32 @added_step(
+; TAILFOLD-SAME: i32 [[N:%.*]], i32 [[STEP_BASE:%.*]], ptr [[P:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[STEP:%.*]] = add i32 [[STEP_BASE]], 1
+; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; TAILFOLD-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; TAILFOLD: [[VECTOR_SCEVCHECK]]:
+; TAILFOLD-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
+; TAILFOLD-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; TAILFOLD-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP0]], 1
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; TAILFOLD-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0
+; TAILFOLD-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[TMP0]])
+; TAILFOLD-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; TAILFOLD-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TAILFOLD-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TAILFOLD: [[PRED_STORE_IF]]:
+; TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
+; TAILFOLD-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TAILFOLD: [[PRED_STORE_CONTINUE]]:
+; TAILFOLD-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TAILFOLD-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_IF1]]:
+; TAILFOLD-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1
+; TAILFOLD-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP8]]
+; TAILFOLD-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
+; TAILFOLD-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4
+; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
+; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP12]], i1 false)
+; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; TAILFOLD-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP3]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[SCALAR_PH]]:
+; TAILFOLD-NEXT: br label %[[LOOP:.*]]
+; TAILFOLD: [[LOOP]]:
+; TAILFOLD-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TAILFOLD-NEXT: [[DERIVED:%.*]] = mul i32 [[IV]], [[STEP]]
+; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; TAILFOLD-NEXT: store i32 [[DERIVED]], ptr [[GEP]], align 4
+; TAILFOLD-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; TAILFOLD-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
+; TAILFOLD-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; TAILFOLD-NEXT: ret i32 [[DERIVED_LCSSA]]
+;
entry:
%step = add i32 %step_base, 1
br label %loop
>From 98ecd59dfd4208611e1afd60f436bb6b4582215c Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 17 Jul 2026 03:22:44 -0700
Subject: [PATCH 2/8] support live-out optimization when tail folding
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +++-
.../LoopVectorize/iv_outside_user.ll | 42 ++++++++++---------
2 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ebd4e91232acb..d19a97b998179 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1018,8 +1018,13 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
VPValue *ResumeTC,
const Loop *L) {
VPValue *Incoming;
- if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming))))
- return nullptr;
+ if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming)))) {
+ VPValue *Mask;
+ if (!match(Op, m_ExtractLane(m_LastActiveLane(m_VPValue(Mask)),
+ m_VPValue(Incoming))) ||
+ !match(Mask, m_HeaderMask()))
+ return nullptr;
+ }
const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
const SCEV *Start, *Step;
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index f2d91f8ebd607..20c204ba660af 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -931,18 +931,12 @@ define i32 @postinc_not_iv_backedge_value(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[K]])
; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
-; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP1:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
-; TAILFOLD-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP2]], i1 false)
-; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: [[TMP1:%.*]] = sub nuw i32 [[K]], 1
+; TAILFOLD-NEXT: [[TMP3:%.*]] = add i32 2, [[TMP1]]
; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
; TAILFOLD: [[FOR_END]]:
; TAILFOLD-NEXT: ret i32 [[TMP3]]
@@ -1919,7 +1913,6 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP1]])
; TAILFOLD-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
@@ -1941,16 +1934,12 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD: [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP12:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
-; TAILFOLD-NEXT: [[TMP13:%.*]] = trunc <2 x i64> [[TMP12]] to <2 x i32>
-; TAILFOLD-NEXT: [[TMP14:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP14]], i1 false)
-; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; TAILFOLD-NEXT: [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = sub nuw i64 [[TMP1]], 1
+; TAILFOLD-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i32
+; TAILFOLD-NEXT: [[TMP15:%.*]] = add i32 1, [[TMP13]]
; TAILFOLD-NEXT: br label %[[EXIT:.*]]
; TAILFOLD: [[EXIT]]:
; TAILFOLD-NEXT: ret i32 [[TMP15]]
@@ -2109,10 +2098,8 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; TAILFOLD-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; TAILFOLD-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP12]], i1 false)
-; TAILFOLD-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; TAILFOLD-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP3]], i64 [[LAST_ACTIVE_LANE]]
+; TAILFOLD-NEXT: [[TMP12:%.*]] = sub nuw i32 [[TMP0]], 1
+; TAILFOLD-NEXT: [[TMP13:%.*]] = mul i32 [[TMP12]], [[STEP]]
; TAILFOLD-NEXT: br label %[[EXIT:.*]]
; TAILFOLD: [[SCALAR_PH]]:
; TAILFOLD-NEXT: br label %[[LOOP:.*]]
@@ -2161,6 +2148,21 @@ define i64 @zext_iv_outside_user() {
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 2147483651
;
+; TAILFOLD-LABEL: define i64 @zext_iv_outside_user() {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; TAILFOLD: [[EXIT]]:
+; TAILFOLD-NEXT: ret i64 2147483651
+;
entry:
br label %loop
>From 9ecb34ce0dd99110f2c281bc2667908395982013 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 13 Aug 2026 01:11:32 -0700
Subject: [PATCH 3/8] rebase and update
---
.../VPlan/buildvector-first-lane-only.ll | 13 +----
.../LoopVectorize/iv_outside_user.ll | 55 +++++++++++++------
2 files changed, 39 insertions(+), 29 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
index b57af70695f6c..6d15c43baac10 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
@@ -4,31 +4,22 @@
define i16 @last_active_lane_live_out(i32 %x) {
; CHECK-LABEL: VPlan for loop in 'last_active_lane_live_out'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
-; CHECK-NEXT: Live-in ir<2> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = step-vector i16
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul vp<[[VP3]]>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not vp<%active.lane.mask>
-; CHECK-NEXT: EMIT vp<%first.inactive.lane> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%last.active.lane> = sub vp<%first.inactive.lane>, ir<1>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = extractelement vp<[[VP5]]>, vp<%last.active.lane>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP7]]> from middle.block)
+; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP2]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 20c204ba660af..cadade6fabcc4 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -39,7 +39,7 @@ define i32 @postinc(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -102,7 +102,7 @@ define i32 @preinc(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -327,7 +327,7 @@ define ptr @both(ptr %p, i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2
; TAILFOLD-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP3]]
@@ -398,7 +398,7 @@ define i32 @multiphi(i32 %k, ptr %p) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -523,14 +523,14 @@ define void @PR30742(ptr %p) {
; TAILFOLD-NEXT: [[START1:%.*]] = add nsw i32 [[TMP2]], -7
; TAILFOLD-NEXT: [[TMP3:%.*]] = add nsw i32 [[TMP2]], -15
; TAILFOLD-NEXT: [[TMP4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP3]], i32 0)
-; TAILFOLD-NEXT: [[TMP5:%.*]] = sub i32 [[TMP2]], [[TMP4]]
-; TAILFOLD-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], -8
+; TAILFOLD-NEXT: [[TMP5:%.*]] = add i32 [[TMP2]], -8
+; TAILFOLD-NEXT: [[TMP6:%.*]] = sub i32 [[TMP5]], [[TMP4]]
; TAILFOLD-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP6]], 3
; TAILFOLD-NEXT: [[TMP8:%.*]] = add nuw nsw i32 [[TMP7]], 1
; TAILFOLD-NEXT: br label %[[VECTOR_PH1:.*]]
; TAILFOLD: [[VECTOR_PH1]]:
; TAILFOLD-NEXT: [[N_RND_UP2:%.*]] = add i32 [[TMP8]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF3:%.*]] = urem i32 [[N_RND_UP2]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF3:%.*]] = and i32 [[N_RND_UP2]], 1
; TAILFOLD-NEXT: [[N_VEC4:%.*]] = sub i32 [[N_RND_UP2]], [[N_MOD_VF3]]
; TAILFOLD-NEXT: [[TMP9:%.*]] = mul i32 [[TMP8]], -8
; TAILFOLD-NEXT: [[TMP10:%.*]] = add i32 [[START1]], [[TMP9]]
@@ -550,14 +550,14 @@ define void @PR30742(ptr %p) {
; TAILFOLD-NEXT: [[START2:%.*]] = add nsw i32 [[TMP14]], -7
; TAILFOLD-NEXT: [[TMP15:%.*]] = add nsw i32 [[TMP14]], -15
; TAILFOLD-NEXT: [[TMP16:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP15]], i32 0)
-; TAILFOLD-NEXT: [[TMP17:%.*]] = sub i32 [[TMP14]], [[TMP16]]
-; TAILFOLD-NEXT: [[TMP18:%.*]] = add i32 [[TMP17]], -8
+; TAILFOLD-NEXT: [[TMP22:%.*]] = add i32 [[TMP14]], -8
+; TAILFOLD-NEXT: [[TMP18:%.*]] = sub i32 [[TMP22]], [[TMP16]]
; TAILFOLD-NEXT: [[TMP19:%.*]] = lshr i32 [[TMP18]], 3
; TAILFOLD-NEXT: [[TMP20:%.*]] = add nuw nsw i32 [[TMP19]], 1
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP20]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -798,7 +798,7 @@ define i32 @postinc_sub(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: [[TMP0:%.*]] = sub i32 [[K]], [[K]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -861,7 +861,7 @@ define i32 @postinc_swapped_ops(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -926,7 +926,7 @@ define i32 @postinc_not_iv_backedge_value(i32 %k) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[K]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -975,6 +975,25 @@ define ptr @postinc_not_iv_backedge_value_ptr_induction(ptr %p.init) {
; CHECK: [[END]]:
; CHECK-NEXT: ret ptr [[TMP2]]
;
+; TAILFOLD-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_induction(
+; TAILFOLD-SAME: ptr [[P_INIT:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[P_END:%.*]] = getelementptr nusw nuw i8, ptr [[P_INIT]], i64 1024
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
+; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1023
+; TAILFOLD-NEXT: br label %[[END:.*]]
+; TAILFOLD: [[END]]:
+; TAILFOLD-NEXT: ret ptr [[TMP2]]
+;
entry:
%p.end = getelementptr nuw nusw i8, ptr %p.init, i64 1024
br label %loop
@@ -1094,7 +1113,7 @@ define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N,
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
@@ -1252,7 +1271,7 @@ define float @fp_postinc_use_fadd_ops_swapped(float %init, ptr noalias nocapture
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
@@ -1410,7 +1429,7 @@ define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N,
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: [[TMP0:%.*]] = sitofp i64 [[N]] to float
; TAILFOLD-NEXT: [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
@@ -1908,7 +1927,7 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
; TAILFOLD: [[VECTOR_BODY]]:
@@ -2066,7 +2085,7 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP0]], 1
-; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 2
+; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0
; TAILFOLD-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
>From 237249aa19b7d393b6ae57ca443b199682b01be1 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 13 Aug 2026 01:12:25 -0700
Subject: [PATCH 4/8] refactor pattern match
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d19a97b998179..34e60fbbab76d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1018,13 +1018,10 @@ static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
VPValue *ResumeTC,
const Loop *L) {
VPValue *Incoming;
- if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming)))) {
- VPValue *Mask;
- if (!match(Op, m_ExtractLane(m_LastActiveLane(m_VPValue(Mask)),
- m_VPValue(Incoming))) ||
- !match(Mask, m_HeaderMask()))
- return nullptr;
- }
+ if (!match(Op, m_CombineOr(m_ExtractLastLaneOfLastPart(m_VPValue(Incoming)),
+ m_ExtractLane(m_LastActiveLane(m_HeaderMask()),
+ m_VPValue(Incoming)))))
+ return nullptr;
const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
const SCEV *Start, *Step;
>From 5e37730028fb50a58f7b85e7a2083c486a237463 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Sun, 23 Aug 2026 23:54:36 -0700
Subject: [PATCH 5/8] rebase and update
---
llvm/test/Transforms/LoopVectorize/iv_outside_user.ll | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index cadade6fabcc4..70b7cef370d23 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -2079,8 +2079,7 @@ define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
; TAILFOLD-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; TAILFOLD: [[VECTOR_SCEVCHECK]]:
-; TAILFOLD-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
-; TAILFOLD-NEXT: [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; TAILFOLD-NEXT: [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[N]], i32 1)
; TAILFOLD-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
; TAILFOLD-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; TAILFOLD: [[VECTOR_PH]]:
>From 7cdceec0874a2c56209ac06ee85f1e004a289c7e Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 24 Aug 2026 01:16:42 -0700
Subject: [PATCH 6/8] update ;RUN:
---
.../LoopVectorize/iv_outside_user.ll | 269 ++++++------------
1 file changed, 89 insertions(+), 180 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 70b7cef370d23..fd8c34b0bdb78 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+" --version 5
-; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=CHECK,VEC %s
-; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=CHECK,INTERLEAVE %s
-; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data < %s | FileCheck --check-prefixes=TAILFOLD %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=COMMON,CHECK,VEC %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=COMMON,CHECK,INTERLEAVE %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data < %s | FileCheck --check-prefixes=COMMON,TAILFOLD %s
define i32 @postinc(i32 %k) {
; CHECK-LABEL: define i32 @postinc(
@@ -130,35 +130,20 @@ for.end:
}
define i32 @constpre() {
-; CHECK-LABEL: define i32 @constpre() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret i32 2
-;
-; TAILFOLD-LABEL: define i32 @constpre() {
-; TAILFOLD-NEXT: [[ENTRY:.*:]]
-; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
-; TAILFOLD: [[VECTOR_PH]]:
-; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
-; TAILFOLD: [[VECTOR_BODY]]:
-; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
-; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
-; TAILFOLD: [[FOR_END]]:
-; TAILFOLD-NEXT: ret i32 2
+; COMMON-LABEL: define i32 @constpre() {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; COMMON-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; COMMON-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[FOR_END:.*]]
+; COMMON: [[FOR_END]]:
+; COMMON-NEXT: ret i32 2
;
entry:
br label %for.body
@@ -174,41 +159,23 @@ for.end:
}
define ptr @geppre(ptr %ptr) {
-; CHECK-LABEL: define ptr @geppre(
-; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: ret ptr [[IND_ESCAPE]]
-;
-; TAILFOLD-LABEL: define ptr @geppre(
-; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
-; TAILFOLD-NEXT: [[ENTRY:.*:]]
-; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
-; TAILFOLD: [[VECTOR_PH]]:
-; TAILFOLD-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
-; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
-; TAILFOLD: [[VECTOR_BODY]]:
-; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
-; TAILFOLD-NEXT: br label %[[FOR_END:.*]]
-; TAILFOLD: [[FOR_END]]:
-; TAILFOLD-NEXT: ret ptr [[IND_ESCAPE]]
+; COMMON-LABEL: define ptr @geppre(
+; COMMON-SAME: ptr [[PTR:%.*]]) {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; COMMON-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; COMMON-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
+; COMMON-NEXT: br label %[[FOR_END:.*]]
+; COMMON: [[FOR_END]]:
+; COMMON-NEXT: ret ptr [[IND_ESCAPE]]
;
entry:
br label %for.body
@@ -956,43 +923,24 @@ for.end:
}
define ptr @postinc_not_iv_backedge_value_ptr_induction(ptr %p.init) {
-; CHECK-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_induction(
-; CHECK-SAME: ptr [[P_INIT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[P_END:%.*]] = getelementptr nusw nuw i8, ptr [[P_INIT]], i64 1024
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1023
-; CHECK-NEXT: br label %[[END:.*]]
-; CHECK: [[END]]:
-; CHECK-NEXT: ret ptr [[TMP2]]
-;
-; TAILFOLD-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_induction(
-; TAILFOLD-SAME: ptr [[P_INIT:%.*]]) {
-; TAILFOLD-NEXT: [[ENTRY:.*:]]
-; TAILFOLD-NEXT: [[P_END:%.*]] = getelementptr nusw nuw i8, ptr [[P_INIT]], i64 1024
-; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
-; TAILFOLD: [[VECTOR_PH]]:
-; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
-; TAILFOLD: [[VECTOR_BODY]]:
-; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
-; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1023
-; TAILFOLD-NEXT: br label %[[END:.*]]
-; TAILFOLD: [[END]]:
-; TAILFOLD-NEXT: ret ptr [[TMP2]]
+; COMMON-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_induction(
+; COMMON-SAME: ptr [[P_INIT:%.*]]) {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[P_END:%.*]] = getelementptr nusw nuw i8, ptr [[P_INIT]], i64 1024
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; COMMON-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMMON-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
+; COMMON-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1023
+; COMMON-NEXT: br label %[[END:.*]]
+; COMMON: [[END]]:
+; COMMON-NEXT: ret ptr [[TMP2]]
;
entry:
%p.end = getelementptr nuw nusw i8, ptr %p.init, i64 1024
@@ -1567,53 +1515,29 @@ e.exit:
}
define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
-; CHECK-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
-; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
-; CHECK-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[OFFSET_IDX]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
-; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[E_EXIT:.*]]
-; CHECK: [[E_EXIT]]:
-; CHECK-NEXT: ret i32 8
-;
-; TAILFOLD-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
-; TAILFOLD-SAME: ptr [[DST:%.*]]) {
-; TAILFOLD-NEXT: [[ENTRY:.*:]]
-; TAILFOLD-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
-; TAILFOLD-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
-; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
-; TAILFOLD: [[VECTOR_PH]]:
-; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
-; TAILFOLD: [[VECTOR_BODY]]:
-; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[TMP0:%.*]] = shl i32 [[INDEX]], 1
-; TAILFOLD-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
-; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
-; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]]
-; TAILFOLD-NEXT: store i16 0, ptr [[TMP2]], align 2
-; TAILFOLD-NEXT: store i16 0, ptr [[TMP3]], align 2
-; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; TAILFOLD-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
-; TAILFOLD-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: br label %[[E_EXIT:.*]]
-; TAILFOLD: [[E_EXIT]]:
-; TAILFOLD-NEXT: ret i32 8
+; COMMON-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
+; COMMON-SAME: ptr [[DST:%.*]]) {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32
+; COMMON-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[TMP0:%.*]] = shl i32 [[INDEX]], 1
+; COMMON-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
+; COMMON-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
+; COMMON-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]]
+; COMMON-NEXT: store i16 0, ptr [[TMP2]], align 2
+; COMMON-NEXT: store i16 0, ptr [[TMP3]], align 2
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; COMMON-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
+; COMMON-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[E_EXIT:.*]]
+; COMMON: [[E_EXIT]]:
+; COMMON-NEXT: ret i32 8
;
entry:
%step.1 = sext i8 0 to i32
@@ -2151,35 +2075,20 @@ exit:
}
define i64 @zext_iv_outside_user() {
-; CHECK-LABEL: define i64 @zext_iv_outside_user() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i64 2147483651
-;
-; TAILFOLD-LABEL: define i64 @zext_iv_outside_user() {
-; TAILFOLD-NEXT: [[ENTRY:.*:]]
-; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
-; TAILFOLD: [[VECTOR_PH]]:
-; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
-; TAILFOLD: [[VECTOR_BODY]]:
-; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; TAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
-; TAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; TAILFOLD: [[MIDDLE_BLOCK]]:
-; TAILFOLD-NEXT: br label %[[EXIT:.*]]
-; TAILFOLD: [[EXIT]]:
-; TAILFOLD-NEXT: ret i64 2147483651
+; COMMON-LABEL: define i64 @zext_iv_outside_user() {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; COMMON-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
+; COMMON-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[EXIT:.*]]
+; COMMON: [[EXIT]]:
+; COMMON-NEXT: ret i64 2147483651
;
entry:
br label %loop
>From 6a5a931f990cd1e85c6a60e513e14fe4f9620810 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 1 Sep 2026 01:56:40 -0700
Subject: [PATCH 7/8] update RUN
---
.../LoopVectorize/iv_outside_user.ll | 514 +++++++++---------
1 file changed, 257 insertions(+), 257 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index fd8c34b0bdb78..905e09f8562fb 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -1,37 +1,37 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+" --version 5
-; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=COMMON,CHECK,VEC %s
-; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=COMMON,CHECK,INTERLEAVE %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=COMMON,NOTAILFOLD,VEC %s
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=COMMON,NOTAILFOLD,INTERLEAVE %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data < %s | FileCheck --check-prefixes=COMMON,TAILFOLD %s
define i32 @postinc(i32 %k) {
-; CHECK-LABEL: define i32 @postinc(
-; CHECK-SAME: i32 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[INC_LCSSA]]
+; NOTAILFOLD-LABEL: define i32 @postinc(
+; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[FOR_BODY:.*]]
+; NOTAILFOLD: [[FOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
+; NOTAILFOLD-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; NOTAILFOLD-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[FOR_END]]:
+; NOTAILFOLD-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: ret i32 [[INC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
@@ -66,35 +66,35 @@ for.end:
}
define i32 @preinc(i32 %k) {
-; CHECK-LABEL: define i32 @preinc(
-; CHECK-SAME: i32 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[N_VEC]], 1
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[INC_PHI_LCSSA]]
+; NOTAILFOLD-LABEL: define i32 @preinc(
+; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[N_VEC]], 1
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[FOR_BODY:.*]]
+; NOTAILFOLD: [[FOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
+; NOTAILFOLD-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; NOTAILFOLD-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[FOR_END]]:
+; NOTAILFOLD-NEXT: [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: ret i32 [[INC_PHI_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @preinc(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
@@ -328,36 +328,36 @@ for.end:
}
define i32 @multiphi(i32 %k, ptr %p) {
-; CHECK-LABEL: define i32 @multiphi(
-; CHECK-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[PHI2:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: store i32 [[PHI2]], ptr [[P]], align 4
-; CHECK-NEXT: ret i32 [[PHI]]
+; NOTAILFOLD-LABEL: define i32 @multiphi(
+; NOTAILFOLD-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[FOR_BODY:.*]]
+; NOTAILFOLD: [[FOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
+; NOTAILFOLD-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; NOTAILFOLD-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[FOR_END]]:
+; NOTAILFOLD-NEXT: [[PHI:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: [[PHI2:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: store i32 [[PHI2]], ptr [[P]], align 4
+; NOTAILFOLD-NEXT: ret i32 [[PHI]]
;
; TAILFOLD-LABEL: define i32 @multiphi(
; TAILFOLD-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
@@ -396,85 +396,85 @@ for.end:
}
define void @PR30742(ptr %p) {
-; CHECK-LABEL: define void @PR30742(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[BB0:.*:]]
-; CHECK-NEXT: br label %[[BB1:.*]]
-; CHECK: [[BB1_LOOPEXIT:.*]]:
-; CHECK-NEXT: br label %[[BB1]]
-; CHECK: [[BB1]]:
-; CHECK-NEXT: [[TMP00:%.*]] = load i32, ptr [[P]], align 16
-; CHECK-NEXT: [[TMP01:%.*]] = sub i32 [[TMP00]], 3
-; CHECK-NEXT: [[TMP02:%.*]] = icmp slt i32 [[TMP01]], 1
-; CHECK-NEXT: [[TMP03:%.*]] = select i1 [[TMP02]], i32 1, i32 [[TMP01]]
-; CHECK-NEXT: [[TMP04:%.*]] = add nsw i32 [[TMP03]], -7
-; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP03]], -15
-; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 0)
-; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP03]], -8
-; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP7]], [[SMIN1]]
-; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 3
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK4:%.*]] = icmp ult i32 [[TMP4]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK4]], label %[[SCALAR_PH2:.*]], label %[[VECTOR_PH4:.*]]
-; CHECK: [[VECTOR_PH4]]:
-; CHECK-NEXT: [[N_MOD_VF6:%.*]] = and i32 [[TMP4]], 1
-; CHECK-NEXT: [[N_VEC7:%.*]] = sub i32 [[TMP4]], [[N_MOD_VF6]]
-; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[N_VEC7]], -8
-; CHECK-NEXT: [[IND_END8:%.*]] = add i32 [[TMP04]], [[TMP5]]
-; CHECK-NEXT: br label %[[VECTOR_BODY7:.*]]
-; CHECK: [[VECTOR_BODY7]]:
-; CHECK-NEXT: [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH4]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY7]] ]
-; CHECK-NEXT: [[INDEX_NEXT11]] = add nuw i32 [[INDEX10]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT11]], [[N_VEC7]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK10:.*]], label %[[VECTOR_BODY7]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK10]]:
-; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[IND_END8]], -8
-; CHECK-NEXT: [[CMP_N12:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC7]]
-; CHECK-NEXT: br i1 [[CMP_N12]], label %[[BB3:.*]], label %[[SCALAR_PH2]]
-; CHECK: [[SCALAR_PH2]]:
-; CHECK-NEXT: [[BC_RESUME_VAL10:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK10]] ], [ [[TMP04]], %[[BB1]] ]
-; CHECK-NEXT: br label %[[BB2:.*]]
-; CHECK: [[BB2]]:
-; CHECK-NEXT: [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL10]], %[[SCALAR_PH2]] ], [ [[TMP06:%.*]], %[[BB2]] ]
-; CHECK-NEXT: [[TMP06]] = add i32 [[TMP05]], -8
-; CHECK-NEXT: [[LOOP1_EC:%.*]] = icmp sgt i32 [[TMP06]], 0
-; CHECK-NEXT: br i1 [[LOOP1_EC]], label %[[BB2]], label %[[BB3]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[BB3]]:
-; CHECK-NEXT: [[IV1_LCSSA:%.*]] = phi i32 [ [[TMP05]], %[[BB2]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK10]] ]
-; CHECK-NEXT: [[TMP09:%.*]] = sub i32 [[TMP00]], 4
-; CHECK-NEXT: [[TMP10:%.*]] = icmp slt i32 [[TMP09]], 1
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 [[TMP09]]
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP11]], -7
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15
-; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0)
-; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP11]], -8
-; CHECK-NEXT: [[TMP21:%.*]] = sub i32 [[TMP18]], [[SMIN]]
-; CHECK-NEXT: [[TMP20:%.*]] = lshr i32 [[TMP21]], 3
-; CHECK-NEXT: [[TMP14:%.*]] = add nuw nsw i32 [[TMP20]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP14]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP14]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP14]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[N_VEC]], -8
-; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[TMP12]], [[TMP16]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP14]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[BB1_LOOPEXIT]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[BB3]] ]
-; CHECK-NEXT: br label %[[BB4:.*]]
-; CHECK: [[BB4]]:
-; CHECK-NEXT: [[IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[BB4]] ]
-; CHECK-NEXT: [[IV2_NEXT]] = add i32 [[IV2]], -8
-; CHECK-NEXT: [[LOOP2_EC:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
-; CHECK-NEXT: br i1 [[LOOP2_EC]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD-LABEL: define void @PR30742(
+; NOTAILFOLD-SAME: ptr [[P:%.*]]) {
+; NOTAILFOLD-NEXT: [[BB0:.*:]]
+; NOTAILFOLD-NEXT: br label %[[BB1:.*]]
+; NOTAILFOLD: [[BB1_LOOPEXIT:.*]]:
+; NOTAILFOLD-NEXT: br label %[[BB1]]
+; NOTAILFOLD: [[BB1]]:
+; NOTAILFOLD-NEXT: [[TMP00:%.*]] = load i32, ptr [[P]], align 16
+; NOTAILFOLD-NEXT: [[TMP01:%.*]] = sub i32 [[TMP00]], 3
+; NOTAILFOLD-NEXT: [[TMP02:%.*]] = icmp slt i32 [[TMP01]], 1
+; NOTAILFOLD-NEXT: [[TMP03:%.*]] = select i1 [[TMP02]], i32 1, i32 [[TMP01]]
+; NOTAILFOLD-NEXT: [[TMP04:%.*]] = add nsw i32 [[TMP03]], -7
+; NOTAILFOLD-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP03]], -15
+; NOTAILFOLD-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 0)
+; NOTAILFOLD-NEXT: [[TMP7:%.*]] = add i32 [[TMP03]], -8
+; NOTAILFOLD-NEXT: [[TMP2:%.*]] = sub i32 [[TMP7]], [[SMIN1]]
+; NOTAILFOLD-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 3
+; NOTAILFOLD-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK4:%.*]] = icmp ult i32 [[TMP4]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK4]], label %[[SCALAR_PH2:.*]], label %[[VECTOR_PH4:.*]]
+; NOTAILFOLD: [[VECTOR_PH4]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF6:%.*]] = and i32 [[TMP4]], 1
+; NOTAILFOLD-NEXT: [[N_VEC7:%.*]] = sub i32 [[TMP4]], [[N_MOD_VF6]]
+; NOTAILFOLD-NEXT: [[TMP5:%.*]] = mul i32 [[N_VEC7]], -8
+; NOTAILFOLD-NEXT: [[IND_END8:%.*]] = add i32 [[TMP04]], [[TMP5]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY7:.*]]
+; NOTAILFOLD: [[VECTOR_BODY7]]:
+; NOTAILFOLD-NEXT: [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH4]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY7]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT11]] = add nuw i32 [[INDEX10]], 2
+; NOTAILFOLD-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT11]], [[N_VEC7]]
+; NOTAILFOLD-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK10:.*]], label %[[VECTOR_BODY7]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK10]]:
+; NOTAILFOLD-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[IND_END8]], -8
+; NOTAILFOLD-NEXT: [[CMP_N12:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC7]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N12]], label %[[BB3:.*]], label %[[SCALAR_PH2]]
+; NOTAILFOLD: [[SCALAR_PH2]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL10:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK10]] ], [ [[TMP04]], %[[BB1]] ]
+; NOTAILFOLD-NEXT: br label %[[BB2:.*]]
+; NOTAILFOLD: [[BB2]]:
+; NOTAILFOLD-NEXT: [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL10]], %[[SCALAR_PH2]] ], [ [[TMP06:%.*]], %[[BB2]] ]
+; NOTAILFOLD-NEXT: [[TMP06]] = add i32 [[TMP05]], -8
+; NOTAILFOLD-NEXT: [[LOOP1_EC:%.*]] = icmp sgt i32 [[TMP06]], 0
+; NOTAILFOLD-NEXT: br i1 [[LOOP1_EC]], label %[[BB2]], label %[[BB3]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[BB3]]:
+; NOTAILFOLD-NEXT: [[IV1_LCSSA:%.*]] = phi i32 [ [[TMP05]], %[[BB2]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK10]] ]
+; NOTAILFOLD-NEXT: [[TMP09:%.*]] = sub i32 [[TMP00]], 4
+; NOTAILFOLD-NEXT: [[TMP10:%.*]] = icmp slt i32 [[TMP09]], 1
+; NOTAILFOLD-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 [[TMP09]]
+; NOTAILFOLD-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP11]], -7
+; NOTAILFOLD-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15
+; NOTAILFOLD-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0)
+; NOTAILFOLD-NEXT: [[TMP18:%.*]] = add i32 [[TMP11]], -8
+; NOTAILFOLD-NEXT: [[TMP21:%.*]] = sub i32 [[TMP18]], [[SMIN]]
+; NOTAILFOLD-NEXT: [[TMP20:%.*]] = lshr i32 [[TMP21]], 3
+; NOTAILFOLD-NEXT: [[TMP14:%.*]] = add nuw nsw i32 [[TMP20]], 1
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP14]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP14]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP14]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: [[TMP16:%.*]] = mul i32 [[N_VEC]], -8
+; NOTAILFOLD-NEXT: [[IND_END:%.*]] = add i32 [[TMP12]], [[TMP16]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP14]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[BB1_LOOPEXIT]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[BB3]] ]
+; NOTAILFOLD-NEXT: br label %[[BB4:.*]]
+; NOTAILFOLD: [[BB4]]:
+; NOTAILFOLD-NEXT: [[IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[BB4]] ]
+; NOTAILFOLD-NEXT: [[IV2_NEXT]] = add i32 [[IV2]], -8
+; NOTAILFOLD-NEXT: [[LOOP2_EC:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; NOTAILFOLD-NEXT: br i1 [[LOOP2_EC]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
;
; TAILFOLD-LABEL: define void @PR30742(
; TAILFOLD-SAME: ptr [[P:%.*]]) {
@@ -729,35 +729,35 @@ exit:
}
define i32 @postinc_sub(i32 %k) {
-; CHECK-LABEL: define i32 @postinc_sub(
-; CHECK-SAME: i32 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[K]], %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INC]] = sub nsw i32 [[INC_PHI]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 0
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[INC_LCSSA]]
+; NOTAILFOLD-LABEL: define i32 @postinc_sub(
+; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = sub i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[K]], %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[FOR_BODY:.*]]
+; NOTAILFOLD: [[FOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INC]] = sub nsw i32 [[INC_PHI]], 1
+; NOTAILFOLD-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 0
+; NOTAILFOLD-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[FOR_END]]:
+; NOTAILFOLD-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: ret i32 [[INC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_sub(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
@@ -793,34 +793,34 @@ for.end:
}
define i32 @postinc_swapped_ops(i32 %k) {
-; CHECK-LABEL: define i32 @postinc_swapped_ops(
-; CHECK-SAME: i32 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INC]] = add nsw i32 1, [[INC_PHI]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[INC_LCSSA]]
+; NOTAILFOLD-LABEL: define i32 @postinc_swapped_ops(
+; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[FOR_BODY:.*]]
+; NOTAILFOLD: [[FOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INC]] = add nsw i32 1, [[INC_PHI]]
+; NOTAILFOLD-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; NOTAILFOLD-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[FOR_END]]:
+; NOTAILFOLD-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: ret i32 [[INC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_swapped_ops(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
@@ -855,37 +855,37 @@ for.end:
}
define i32 @postinc_not_iv_backedge_value(i32 %k) {
-; CHECK-LABEL: define i32 @postinc_not_iv_backedge_value(
-; CHECK-SAME: i32 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP1:%.*]] = sub nuw i32 [[N_VEC]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 2, [[TMP1]]
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; CHECK-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: ret i32 [[INC_2_LCSSA]]
+; NOTAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
+; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[N_MOD_VF:%.*]] = and i32 [[K]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[TMP1:%.*]] = sub nuw i32 [[N_VEC]], 1
+; NOTAILFOLD-NEXT: [[TMP2:%.*]] = add i32 2, [[TMP1]]
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[FOR_BODY:.*]]
+; NOTAILFOLD: [[FOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
+; NOTAILFOLD-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
+; NOTAILFOLD-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; NOTAILFOLD-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[FOR_END]]:
+; NOTAILFOLD-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: ret i32 [[INC_2_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
@@ -1401,7 +1401,7 @@ define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N,
; TAILFOLD: [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
; TAILFOLD-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
-; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float -1.000000e+00, [[FPINC]]
+; TAILFOLD-NEXT: [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; TAILFOLD-NEXT: [[TMP12:%.*]] = fsub fast float [[TMP5]], [[TMP11]]
; TAILFOLD-NEXT: store float [[TMP12]], ptr [[TMP10]], align 4
; TAILFOLD-NEXT: br label %[[PRED_STORE_CONTINUE2]]
>From 1169b0b780dd07c171e5b9f9b5e1d63c51fe5029 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 1 Sep 2026 05:28:58 -0700
Subject: [PATCH 8/8] add test for ptr type
---
.../LoopVectorize/iv_outside_user.ll | 74 +++++++++++++++++++
1 file changed, 74 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 905e09f8562fb..175b5c7b075a0 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -957,6 +957,80 @@ end:
ret ptr %ret
}
+define ptr @postinc_not_iv_backedge_value_ptr_runtime_tc(ptr %p.init, i64 %n) {
+; NOTAILFOLD-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_runtime_tc(
+; NOTAILFOLD-SAME: ptr [[P_INIT:%.*]], i64 [[N:%.*]]) {
+; NOTAILFOLD-NEXT: [[ENTRY:.*]]:
+; NOTAILFOLD-NEXT: [[P_END:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 [[N]]
+; NOTAILFOLD-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; NOTAILFOLD-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NOTAILFOLD: [[VECTOR_PH]]:
+; NOTAILFOLD-NEXT: [[TMP0:%.*]] = and i64 [[N]], 1
+; NOTAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; NOTAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 [[N_VEC]]
+; NOTAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; NOTAILFOLD: [[VECTOR_BODY]]:
+; NOTAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NOTAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; NOTAILFOLD-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[MIDDLE_BLOCK]]:
+; NOTAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
+; NOTAILFOLD-NEXT: [[TMP4:%.*]] = sub nuw i64 [[N_VEC]], 1
+; NOTAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP4]]
+; NOTAILFOLD-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NOTAILFOLD-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
+; NOTAILFOLD: [[SCALAR_PH]]:
+; NOTAILFOLD-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[P_INIT]], %[[ENTRY]] ]
+; NOTAILFOLD-NEXT: br label %[[LOOP:.*]]
+; NOTAILFOLD: [[LOOP]]:
+; NOTAILFOLD-NEXT: [[P_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; NOTAILFOLD-NEXT: [[P_NEXT]] = getelementptr nuw i8, ptr [[P_IV]], i64 1
+; NOTAILFOLD-NEXT: [[RET:%.*]] = getelementptr i8, ptr [[P_IV]], i64 2
+; NOTAILFOLD-NEXT: [[EC:%.*]] = icmp eq ptr [[P_NEXT]], [[P_END]]
+; NOTAILFOLD-NEXT: br i1 [[EC]], label %[[END]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
+; NOTAILFOLD: [[END]]:
+; NOTAILFOLD-NEXT: [[RET_LCSSA:%.*]] = phi ptr [ [[RET]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
+; NOTAILFOLD-NEXT: ret ptr [[RET_LCSSA]]
+;
+; TAILFOLD-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_runtime_tc(
+; TAILFOLD-SAME: ptr [[P_INIT:%.*]], i64 [[N:%.*]]) {
+; TAILFOLD-NEXT: [[ENTRY:.*:]]
+; TAILFOLD-NEXT: [[P_END:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 [[N]]
+; TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; TAILFOLD: [[VECTOR_PH]]:
+; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1
+; TAILFOLD-NEXT: [[TMP0:%.*]] = and i64 [[N_RND_UP]], 1
+; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
+; TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; TAILFOLD: [[VECTOR_BODY]]:
+; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TAILFOLD-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; TAILFOLD: [[MIDDLE_BLOCK]]:
+; TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
+; TAILFOLD-NEXT: [[TMP3:%.*]] = sub nuw i64 [[N]], 1
+; TAILFOLD-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 [[TMP3]]
+; TAILFOLD-NEXT: br label %[[END:.*]]
+; TAILFOLD: [[END]]:
+; TAILFOLD-NEXT: ret ptr [[TMP4]]
+;
+entry:
+ %p.end = getelementptr i8, ptr %p.init, i64 %n
+ br label %loop
+
+loop:
+ %p.iv = phi ptr [ %p.init, %entry ], [ %p.next, %loop ]
+ %p.next = getelementptr nuw i8, ptr %p.iv, i64 1
+ %ret = getelementptr i8, ptr %p.iv, i64 2
+ %ec = icmp eq ptr %p.next, %p.end
+ br i1 %ec, label %end, label %loop
+
+end:
+ ret ptr %ret
+}
+
define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
; VEC-LABEL: define float @fp_postinc_use_fadd(
; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
More information about the llvm-commits
mailing list