[llvm] 508bdbc - [LV][NFC] Pre-commit SCEV check tests for PR #216975 (#217037)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 23:30:48 PDT 2026
Author: David Sherwood
Date: 2026-08-19T07:30:43+01:00
New Revision: 508bdbc10203a4b1365078fae0ae1259ea979cf1
URL: https://github.com/llvm/llvm-project/commit/508bdbc10203a4b1365078fae0ae1259ea979cf1
DIFF: https://github.com/llvm/llvm-project/commit/508bdbc10203a4b1365078fae0ae1259ea979cf1.diff
LOG: [LV][NFC] Pre-commit SCEV check tests for PR #216975 (#217037)
Added:
Modified:
llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
Removed:
################################################################################
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 898a5d9e8a461..aa192fc9d4d67 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -15,50 +15,50 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
-; DEFAULT-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP9:%.*]] = shl i64 [[TMP3]], 4
-; DEFAULT-NEXT: [[TMP27:%.*]] = add i64 [[TMP9]], -1
-; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[DST1]], [[SRC2]]
-; DEFAULT-NEXT: [[TMP14:%.*]] = sub i64 [[TMP6]], 1
-; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP27]]
+; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; DEFAULT-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], -1
+; DEFAULT-NEXT: [[TMP4:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; DEFAULT-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1
+; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP3]]
; DEFAULT-NEXT: br i1 [[DIFF_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP1]], 4
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP0]], [[TMP8]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 4
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[TMP0]], [[TMP7]]
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK3]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[TMP13:%.*]] = shl nuw i64 [[TMP1]], 3
-; DEFAULT-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP13]], 1
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP10]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP6]], 3
+; DEFAULT-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP9]]
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP11:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP10:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP13]]
-; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
-; DEFAULT-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP15]], align 1
-; DEFAULT-NEXT: [[TMP16:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i16>
-; DEFAULT-NEXT: [[TMP17:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i16>
-; DEFAULT-NEXT: [[TMP18:%.*]] = mul <vscale x 8 x i16> [[TMP16]], [[TMP11]]
-; DEFAULT-NEXT: [[TMP19:%.*]] = mul <vscale x 8 x i16> [[TMP17]], [[TMP11]]
-; DEFAULT-NEXT: [[TMP20:%.*]] = or <vscale x 8 x i16> [[TMP18]], [[TMP16]]
-; DEFAULT-NEXT: [[TMP21:%.*]] = or <vscale x 8 x i16> [[TMP19]], [[TMP17]]
-; DEFAULT-NEXT: [[TMP22:%.*]] = lshr <vscale x 8 x i16> [[TMP20]], splat (i16 1)
-; DEFAULT-NEXT: [[TMP23:%.*]] = lshr <vscale x 8 x i16> [[TMP21]], splat (i16 1)
-; DEFAULT-NEXT: [[TMP24:%.*]] = trunc <vscale x 8 x i16> [[TMP22]] to <vscale x 8 x i8>
-; DEFAULT-NEXT: [[TMP25:%.*]] = trunc <vscale x 8 x i16> [[TMP23]] to <vscale x 8 x i8>
-; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; DEFAULT-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[TMP26]], i64 [[TMP13]]
-; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP24]], ptr [[TMP26]], align 1
-; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP25]], ptr [[TMP29]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
-; DEFAULT-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; DEFAULT-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP8]]
+; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; DEFAULT-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; DEFAULT-NEXT: [[TMP13:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP14:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i16>
+; DEFAULT-NEXT: [[TMP15:%.*]] = mul <vscale x 8 x i16> [[TMP13]], [[TMP10]]
+; DEFAULT-NEXT: [[TMP16:%.*]] = mul <vscale x 8 x i16> [[TMP14]], [[TMP10]]
+; DEFAULT-NEXT: [[TMP17:%.*]] = or <vscale x 8 x i16> [[TMP15]], [[TMP13]]
+; DEFAULT-NEXT: [[TMP18:%.*]] = or <vscale x 8 x i16> [[TMP16]], [[TMP14]]
+; DEFAULT-NEXT: [[TMP19:%.*]] = lshr <vscale x 8 x i16> [[TMP17]], splat (i16 1)
+; DEFAULT-NEXT: [[TMP20:%.*]] = lshr <vscale x 8 x i16> [[TMP18]], splat (i16 1)
+; DEFAULT-NEXT: [[TMP21:%.*]] = trunc <vscale x 8 x i16> [[TMP19]] to <vscale x 8 x i8>
+; DEFAULT-NEXT: [[TMP22:%.*]] = trunc <vscale x 8 x i16> [[TMP20]] to <vscale x 8 x i8>
+; DEFAULT-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP23]], i64 [[TMP8]]
+; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP21]], ptr [[TMP23]], align 1
+; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP22]], ptr [[TMP24]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; DEFAULT-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -67,31 +67,31 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_PH]]:
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; DEFAULT-NEXT: [[N_MOD_VF5:%.*]] = and i64 [[TMP0]], 7
-; DEFAULT-NEXT: [[N_VEC6:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF5]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT7]], <8 x i32> poison, <8 x i32> zeroinitializer
-; DEFAULT-NEXT: [[TMP28:%.*]] = trunc <8 x i32> [[BROADCAST_SPLAT8]] to <8 x i16>
+; DEFAULT-NEXT: [[TMP26:%.*]] = and i64 [[TMP0]], 7
+; DEFAULT-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[TMP26]]
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT6]], <8 x i32> poison, <8 x i32> zeroinitializer
+; DEFAULT-NEXT: [[TMP27:%.*]] = trunc <8 x i32> [[BROADCAST_SPLAT7]] to <8 x i16>
; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX9]]
-; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i8>, ptr [[TMP36]], align 1
-; DEFAULT-NEXT: [[TMP34:%.*]] = zext <8 x i8> [[WIDE_LOAD10]] to <8 x i16>
-; DEFAULT-NEXT: [[TMP35:%.*]] = mul <8 x i16> [[TMP34]], [[TMP28]]
-; DEFAULT-NEXT: [[TMP31:%.*]] = or <8 x i16> [[TMP35]], [[TMP34]]
+; DEFAULT-NEXT: [[INDEX8:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX8]]
+; DEFAULT-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i8>, ptr [[TMP28]], align 1
+; DEFAULT-NEXT: [[TMP29:%.*]] = zext <8 x i8> [[WIDE_LOAD9]] to <8 x i16>
+; DEFAULT-NEXT: [[TMP30:%.*]] = mul <8 x i16> [[TMP29]], [[TMP27]]
+; DEFAULT-NEXT: [[TMP31:%.*]] = or <8 x i16> [[TMP30]], [[TMP29]]
; DEFAULT-NEXT: [[TMP32:%.*]] = lshr <8 x i16> [[TMP31]], splat (i16 1)
; DEFAULT-NEXT: [[TMP33:%.*]] = trunc <8 x i16> [[TMP32]] to <8 x i8>
-; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX9]]
-; DEFAULT-NEXT: store <8 x i8> [[TMP33]], ptr [[TMP42]], align 1
-; DEFAULT-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], 8
-; DEFAULT-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC6]]
-; DEFAULT-NEXT: br i1 [[TMP43]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX8]]
+; DEFAULT-NEXT: store <8 x i8> [[TMP33]], ptr [[TMP34]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX8]], 8
+; DEFAULT-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC5]]
+; DEFAULT-NEXT: br i1 [[TMP35]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC6]]
-; DEFAULT-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; DEFAULT-NEXT: [[CMP_N11:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]
+; DEFAULT-NEXT: br i1 [[CMP_N11]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; DEFAULT: [[VEC_EPILOG_SCALAR_PH]]:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ], [ 0, %[[ITER_CHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -121,36 +121,36 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; PRED: [[VECTOR_MEMCHECK]]:
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; PRED-NEXT: [[TMP2:%.*]] = shl i64 [[TMP1]], 3
-; PRED-NEXT: [[TMP7:%.*]] = add i64 [[TMP2]], -1
-; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
-; PRED-NEXT: [[TMP6:%.*]] = sub i64 [[TMP3]], 1
-; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP7]]
+; PRED-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], -1
+; PRED-NEXT: [[TMP4:%.*]] = sub i64 [[DST1]], [[SRC2]]
+; PRED-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1
+; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP3]]
; PRED-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
-; PRED-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
+; PRED-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; PRED-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 3
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; PRED-NEXT: [[TMP14:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
+; PRED-NEXT: [[TMP8:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PRED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr align 1 [[TMP12]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i8> poison)
-; PRED-NEXT: [[TMP8:%.*]] = zext <vscale x 8 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 8 x i16>
-; PRED-NEXT: [[TMP9:%.*]] = mul <vscale x 8 x i16> [[TMP8]], [[TMP14]]
-; PRED-NEXT: [[TMP10:%.*]] = or <vscale x 8 x i16> [[TMP9]], [[TMP8]]
-; PRED-NEXT: [[TMP11:%.*]] = lshr <vscale x 8 x i16> [[TMP10]], splat (i16 1)
-; PRED-NEXT: [[TMP13:%.*]] = trunc <vscale x 8 x i16> [[TMP11]] to <vscale x 8 x i8>
-; PRED-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; PRED-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP13]], ptr align 1 [[TMP18]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
+; PRED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr align 1 [[TMP9]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i8> poison)
+; PRED-NEXT: [[TMP10:%.*]] = zext <vscale x 8 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 8 x i16>
+; PRED-NEXT: [[TMP11:%.*]] = mul <vscale x 8 x i16> [[TMP10]], [[TMP8]]
+; PRED-NEXT: [[TMP12:%.*]] = or <vscale x 8 x i16> [[TMP11]], [[TMP10]]
+; PRED-NEXT: [[TMP13:%.*]] = lshr <vscale x 8 x i16> [[TMP12]], splat (i16 1)
+; PRED-NEXT: [[TMP14:%.*]] = trunc <vscale x 8 x i16> [[TMP13]] to <vscale x 8 x i8>
+; PRED-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
+; PRED-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr align 1 [[TMP15]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
-; PRED-NEXT: [[TMP19:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PRED-NEXT: [[TMP20:%.*]] = xor i1 [[TMP19]], true
-; PRED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PRED-NEXT: [[TMP16:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; PRED-NEXT: [[TMP17:%.*]] = xor i1 [[TMP16]], true
+; PRED-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
; PRED-NEXT: br label %[[EXIT:.*]]
; PRED: [[SCALAR_PH]]:
@@ -222,34 +222,34 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
; DEFAULT-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; DEFAULT-NEXT: [[TMP13:%.*]] = and i64 [[TMP0]], 3
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP13:%.*]] = trunc i64 [[INDEX]] to i32
-; DEFAULT-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
-; DEFAULT-NEXT: [[TMP22:%.*]] = add i32 [[TMP13]], 2
-; DEFAULT-NEXT: [[TMP25:%.*]] = add i32 [[TMP13]], 3
-; DEFAULT-NEXT: [[TMP15:%.*]] = mul i32 [[MUL_X]], [[TMP13]]
-; DEFAULT-NEXT: [[TMP16:%.*]] = mul i32 [[MUL_X]], [[TMP14]]
-; DEFAULT-NEXT: [[TMP26:%.*]] = mul i32 [[MUL_X]], [[TMP22]]
-; DEFAULT-NEXT: [[TMP29:%.*]] = mul i32 [[MUL_X]], [[TMP25]]
-; DEFAULT-NEXT: [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
-; DEFAULT-NEXT: [[TMP18:%.*]] = zext i32 [[TMP16]] to i64
-; DEFAULT-NEXT: [[TMP23:%.*]] = zext i32 [[TMP26]] to i64
-; DEFAULT-NEXT: [[TMP24:%.*]] = zext i32 [[TMP29]] to i64
-; DEFAULT-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP17]]
-; DEFAULT-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP18]]
+; DEFAULT-NEXT: [[TMP14:%.*]] = trunc i64 [[INDEX]] to i32
+; DEFAULT-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 1
+; DEFAULT-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], 2
+; DEFAULT-NEXT: [[TMP17:%.*]] = add i32 [[TMP14]], 3
+; DEFAULT-NEXT: [[TMP18:%.*]] = mul i32 [[MUL_X]], [[TMP14]]
+; DEFAULT-NEXT: [[TMP19:%.*]] = mul i32 [[MUL_X]], [[TMP15]]
+; DEFAULT-NEXT: [[TMP20:%.*]] = mul i32 [[MUL_X]], [[TMP16]]
+; DEFAULT-NEXT: [[TMP21:%.*]] = mul i32 [[MUL_X]], [[TMP17]]
+; DEFAULT-NEXT: [[TMP22:%.*]] = zext i32 [[TMP18]] to i64
+; DEFAULT-NEXT: [[TMP23:%.*]] = zext i32 [[TMP19]] to i64
+; DEFAULT-NEXT: [[TMP24:%.*]] = zext i32 [[TMP20]] to i64
+; DEFAULT-NEXT: [[TMP25:%.*]] = zext i32 [[TMP21]] to i64
+; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP22]]
; DEFAULT-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP23]]
; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP24]]
-; DEFAULT-NEXT: store i32 1, ptr [[TMP19]], align 4
-; DEFAULT-NEXT: store i32 1, ptr [[TMP20]], align 4
+; DEFAULT-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP25]]
+; DEFAULT-NEXT: store i32 1, ptr [[TMP26]], align 4
; DEFAULT-NEXT: store i32 1, ptr [[TMP27]], align 4
; DEFAULT-NEXT: store i32 1, ptr [[TMP28]], align 4
+; DEFAULT-NEXT: store i32 1, ptr [[TMP29]], align 4
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DEFAULT-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; DEFAULT-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -334,44 +334,44 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP13:%.*]] = or i1 [[TMP9]], [[TMP12]]
; DEFAULT-NEXT: br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT: [[TMP14:%.*]] = trunc i64 [[N_VEC]] to i32
+; DEFAULT-NEXT: [[TMP14:%.*]] = and i64 [[TMP0]], 3
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP14]]
+; DEFAULT-NEXT: [[TMP15:%.*]] = trunc i64 [[N_VEC]] to i32
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
-; DEFAULT-NEXT: [[TMP15:%.*]] = add i32 [[OFFSET_IDX]], 1
-; DEFAULT-NEXT: [[TMP26:%.*]] = add i32 [[OFFSET_IDX]], 2
-; DEFAULT-NEXT: [[TMP27:%.*]] = add i32 [[OFFSET_IDX]], 3
-; DEFAULT-NEXT: [[TMP17:%.*]] = add i32 [[OFFSET_IDX]], 1
-; DEFAULT-NEXT: [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], 2
-; DEFAULT-NEXT: [[TMP31:%.*]] = add i32 [[OFFSET_IDX]], 3
-; DEFAULT-NEXT: [[TMP18:%.*]] = mul i32 [[MUL]], [[OFFSET_IDX]]
-; DEFAULT-NEXT: [[TMP19:%.*]] = mul i32 [[MUL]], [[TMP17]]
-; DEFAULT-NEXT: [[TMP34:%.*]] = mul i32 [[MUL]], [[TMP30]]
-; DEFAULT-NEXT: [[TMP25:%.*]] = mul i32 [[MUL]], [[TMP31]]
-; DEFAULT-NEXT: [[TMP20:%.*]] = zext i32 [[TMP18]] to i64
-; DEFAULT-NEXT: [[TMP21:%.*]] = zext i32 [[TMP19]] to i64
-; DEFAULT-NEXT: [[TMP28:%.*]] = zext i32 [[TMP34]] to i64
+; DEFAULT-NEXT: [[TMP16:%.*]] = trunc i64 [[INDEX]] to i32
+; DEFAULT-NEXT: [[TMP17:%.*]] = add i32 [[TMP16]], 1
+; DEFAULT-NEXT: [[TMP18:%.*]] = add i32 [[TMP16]], 2
+; DEFAULT-NEXT: [[TMP19:%.*]] = add i32 [[TMP16]], 3
+; DEFAULT-NEXT: [[TMP20:%.*]] = add i32 [[TMP16]], 1
+; DEFAULT-NEXT: [[TMP21:%.*]] = add i32 [[TMP16]], 2
+; DEFAULT-NEXT: [[TMP22:%.*]] = add i32 [[TMP16]], 3
+; DEFAULT-NEXT: [[TMP23:%.*]] = mul i32 [[MUL]], [[TMP16]]
+; DEFAULT-NEXT: [[TMP24:%.*]] = mul i32 [[MUL]], [[TMP20]]
+; DEFAULT-NEXT: [[TMP25:%.*]] = mul i32 [[MUL]], [[TMP21]]
+; DEFAULT-NEXT: [[TMP26:%.*]] = mul i32 [[MUL]], [[TMP22]]
+; DEFAULT-NEXT: [[TMP27:%.*]] = zext i32 [[TMP23]] to i64
+; DEFAULT-NEXT: [[TMP28:%.*]] = zext i32 [[TMP24]] to i64
; DEFAULT-NEXT: [[TMP29:%.*]] = zext i32 [[TMP25]] to i64
-; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP20]]
-; DEFAULT-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP21]]
+; DEFAULT-NEXT: [[TMP30:%.*]] = zext i32 [[TMP26]] to i64
+; DEFAULT-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP27]]
; DEFAULT-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP28]]
; DEFAULT-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP29]]
-; DEFAULT-NEXT: store i32 [[OFFSET_IDX]], ptr [[TMP22]], align 4
-; DEFAULT-NEXT: store i32 [[TMP15]], ptr [[TMP23]], align 4
-; DEFAULT-NEXT: store i32 [[TMP26]], ptr [[TMP32]], align 4
-; DEFAULT-NEXT: store i32 [[TMP27]], ptr [[TMP33]], align 4
+; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP30]]
+; DEFAULT-NEXT: store i32 [[TMP16]], ptr [[TMP31]], align 4
+; DEFAULT-NEXT: store i32 [[TMP17]], ptr [[TMP32]], align 4
+; DEFAULT-NEXT: store i32 [[TMP18]], ptr [[TMP33]], align 4
+; DEFAULT-NEXT: store i32 [[TMP19]], ptr [[TMP34]], align 4
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DEFAULT-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; DEFAULT-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
@@ -391,13 +391,13 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; PRED-LABEL: define void @trunc_ivs_and_store(
; PRED-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; PRED-NEXT: [[ENTRY:.*]]:
-; PRED-NEXT: [[TMP1:%.*]] = mul i32 [[X]], [[X]]
+; PRED-NEXT: [[MUL:%.*]] = mul i32 [[X]], [[X]]
; PRED-NEXT: br label %[[LOOP:.*]]
; PRED: [[LOOP]]:
; PRED-NEXT: [[IV_1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
; PRED-NEXT: [[IV_2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
; PRED-NEXT: [[IV_1_TRUNC:%.*]] = trunc i64 [[IV_1]] to i32
-; PRED-NEXT: [[IV_1_MUL:%.*]] = mul i32 [[TMP1]], [[IV_1_TRUNC]]
+; PRED-NEXT: [[IV_1_MUL:%.*]] = mul i32 [[MUL]], [[IV_1_TRUNC]]
; PRED-NEXT: [[IV_2_NEXT]] = add i32 [[IV_2]], 1
; PRED-NEXT: [[MUL_EXT:%.*]] = zext i32 [[IV_1_MUL]] to i64
; PRED-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[MUL_EXT]]
@@ -455,44 +455,44 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
; DEFAULT-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 3
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT: [[TMP13:%.*]] = trunc i64 [[N_VEC]] to i32
+; DEFAULT-NEXT: [[TMP13:%.*]] = and i64 [[TMP0]], 3
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
+; DEFAULT-NEXT: [[TMP14:%.*]] = trunc i64 [[N_VEC]] to i32
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
-; DEFAULT-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], 1
-; DEFAULT-NEXT: [[TMP25:%.*]] = add i32 [[OFFSET_IDX]], 2
-; DEFAULT-NEXT: [[TMP26:%.*]] = add i32 [[OFFSET_IDX]], 3
-; DEFAULT-NEXT: [[TMP16:%.*]] = add i32 [[OFFSET_IDX]], 1
-; DEFAULT-NEXT: [[TMP29:%.*]] = add i32 [[OFFSET_IDX]], 2
-; DEFAULT-NEXT: [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], 3
-; DEFAULT-NEXT: [[TMP17:%.*]] = mul i32 [[ADD]], [[OFFSET_IDX]]
-; DEFAULT-NEXT: [[TMP18:%.*]] = mul i32 [[ADD]], [[TMP16]]
-; DEFAULT-NEXT: [[TMP33:%.*]] = mul i32 [[ADD]], [[TMP29]]
-; DEFAULT-NEXT: [[TMP24:%.*]] = mul i32 [[ADD]], [[TMP30]]
-; DEFAULT-NEXT: [[TMP19:%.*]] = zext i32 [[TMP17]] to i64
-; DEFAULT-NEXT: [[TMP20:%.*]] = zext i32 [[TMP18]] to i64
-; DEFAULT-NEXT: [[TMP27:%.*]] = zext i32 [[TMP33]] to i64
+; DEFAULT-NEXT: [[TMP15:%.*]] = trunc i64 [[INDEX]] to i32
+; DEFAULT-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 1
+; DEFAULT-NEXT: [[TMP17:%.*]] = add i32 [[TMP15]], 2
+; DEFAULT-NEXT: [[TMP18:%.*]] = add i32 [[TMP15]], 3
+; DEFAULT-NEXT: [[TMP19:%.*]] = add i32 [[TMP15]], 1
+; DEFAULT-NEXT: [[TMP20:%.*]] = add i32 [[TMP15]], 2
+; DEFAULT-NEXT: [[TMP21:%.*]] = add i32 [[TMP15]], 3
+; DEFAULT-NEXT: [[TMP22:%.*]] = mul i32 [[ADD]], [[TMP15]]
+; DEFAULT-NEXT: [[TMP23:%.*]] = mul i32 [[ADD]], [[TMP19]]
+; DEFAULT-NEXT: [[TMP24:%.*]] = mul i32 [[ADD]], [[TMP20]]
+; DEFAULT-NEXT: [[TMP25:%.*]] = mul i32 [[ADD]], [[TMP21]]
+; DEFAULT-NEXT: [[TMP26:%.*]] = zext i32 [[TMP22]] to i64
+; DEFAULT-NEXT: [[TMP27:%.*]] = zext i32 [[TMP23]] to i64
; DEFAULT-NEXT: [[TMP28:%.*]] = zext i32 [[TMP24]] to i64
-; DEFAULT-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP19]]
-; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP20]]
+; DEFAULT-NEXT: [[TMP29:%.*]] = zext i32 [[TMP25]] to i64
+; DEFAULT-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP26]]
; DEFAULT-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP27]]
; DEFAULT-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP28]]
-; DEFAULT-NEXT: store i32 [[OFFSET_IDX]], ptr [[TMP21]], align 4
-; DEFAULT-NEXT: store i32 [[TMP14]], ptr [[TMP22]], align 4
-; DEFAULT-NEXT: store i32 [[TMP25]], ptr [[TMP31]], align 4
-; DEFAULT-NEXT: store i32 [[TMP26]], ptr [[TMP32]], align 4
+; DEFAULT-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP29]]
+; DEFAULT-NEXT: store i32 [[TMP15]], ptr [[TMP30]], align 4
+; DEFAULT-NEXT: store i32 [[TMP16]], ptr [[TMP31]], align 4
+; DEFAULT-NEXT: store i32 [[TMP17]], ptr [[TMP32]], align 4
+; DEFAULT-NEXT: store i32 [[TMP18]], ptr [[TMP33]], align 4
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DEFAULT-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; DEFAULT-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
; DEFAULT-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
@@ -551,48 +551,48 @@ exit:
ret void
}
-define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
-; DEFAULT-LABEL: define void @exit_cond_zext_iv(
+define void @exit_cond_zext_iv_store32(ptr %dst, i64 %N) {
+; DEFAULT-LABEL: define void @exit_cond_zext_iv_store32(
; DEFAULT-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; DEFAULT-NEXT: [[ENTRY:.*]]:
-; DEFAULT-NEXT: [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 4
+; DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; DEFAULT: [[VECTOR_SCEVCHECK]]:
; DEFAULT-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1
-; DEFAULT-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; DEFAULT-NEXT: [[TMP2:%.*]] = add i32 1, [[TMP1]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 1
-; DEFAULT-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[TMP0]], 4294967295
-; DEFAULT-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; DEFAULT-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -1
+; DEFAULT-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; DEFAULT-NEXT: [[TMP3:%.*]] = add i32 1, [[TMP2]]
+; DEFAULT-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 1
+; DEFAULT-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; DEFAULT-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; DEFAULT-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = and i64 [[UMAX1]], 3
-; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = trunc i64 [[N_VEC]] to i32
+; DEFAULT-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 3
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP7]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = trunc i64 [[N_VEC]] to i32
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; DEFAULT-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 2
-; DEFAULT-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 3
-; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[INDEX]], i32 2
-; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP7]], i32 2
-; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP11]], i32 2
-; DEFAULT-NEXT: [[TMP13:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP14]], i32 2
-; DEFAULT-NEXT: store i32 0, ptr [[TMP8]], align 8
-; DEFAULT-NEXT: store i32 0, ptr [[TMP9]], align 8
+; DEFAULT-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; DEFAULT-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 2
+; DEFAULT-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 3
+; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[INDEX]], i32 2
+; DEFAULT-NEXT: [[TMP13:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP9]], i32 2
+; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP10]], i32 2
+; DEFAULT-NEXT: [[TMP15:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP11]], i32 2
; DEFAULT-NEXT: store i32 0, ptr [[TMP12]], align 8
; DEFAULT-NEXT: store i32 0, ptr [[TMP13]], align 8
+; DEFAULT-NEXT: store i32 0, ptr [[TMP14]], align 8
+; DEFAULT-NEXT: store i32 0, ptr [[TMP15]], align 8
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; DEFAULT-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; DEFAULT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]
+; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; DEFAULT: [[SCALAR_PH]]:
-; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; DEFAULT-NEXT: br label %[[LOOP:.*]]
; DEFAULT: [[LOOP]]:
@@ -607,51 +607,51 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
; DEFAULT: [[EXIT]]:
; DEFAULT-NEXT: ret void
;
-; PRED-LABEL: define void @exit_cond_zext_iv(
+; PRED-LABEL: define void @exit_cond_zext_iv_store32(
; PRED-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; PRED-NEXT: [[ENTRY:.*:]]
-; PRED-NEXT: [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; PRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; PRED-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; PRED: [[VECTOR_SCEVCHECK]]:
; PRED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; PRED-NEXT: [[TMP0:%.*]] = add i64 [[UMAX]], -1
-; PRED-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; PRED-NEXT: [[TMP2:%.*]] = add i32 1, [[TMP1]]
-; PRED-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 1
-; PRED-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[TMP0]], 4294967295
-; PRED-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
-; PRED-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; PRED-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -1
+; PRED-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; PRED-NEXT: [[TMP3:%.*]] = add i32 1, [[TMP2]]
+; PRED-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 1
+; PRED-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; PRED-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; PRED-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; PRED: [[VECTOR_PH]]:
-; PRED-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX1]], 1
-; PRED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
-; PRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[UMAX1]], 1
+; PRED-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 1
+; PRED-NEXT: [[TMP7:%.*]] = and i64 [[N_RND_UP]], 1
+; PRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP7]]
+; PRED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
-; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
-; PRED-NEXT: [[VEC_IV:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; PRED-NEXT: [[TMP6:%.*]] = icmp ule <2 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; PRED-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
-; PRED-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; PRED-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; PRED-NEXT: [[TMP8:%.*]] = icmp ule <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PRED-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP8]], i64 0
+; PRED-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; PRED: [[PRED_STORE_IF]]:
-; PRED-NEXT: [[TMP9:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[INDEX]], i32 2
-; PRED-NEXT: store i32 0, ptr [[TMP9]], align 8
+; PRED-NEXT: [[TMP10:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[INDEX]], i32 2
+; PRED-NEXT: store i32 0, ptr [[TMP10]], align 8
; PRED-NEXT: br label %[[PRED_STORE_CONTINUE]]
; PRED: [[PRED_STORE_CONTINUE]]:
-; PRED-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1
-; PRED-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5]]
-; PRED: [[PRED_STORE_IF4]]:
-; PRED-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
-; PRED-NEXT: [[TMP12:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP11]], i32 2
-; PRED-NEXT: store i32 0, ptr [[TMP12]], align 8
-; PRED-NEXT: br label %[[PRED_STORE_CONTINUE5]]
-; PRED: [[PRED_STORE_CONTINUE5]]:
+; PRED-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i64 1
+; PRED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; PRED: [[PRED_STORE_IF1]]:
+; PRED-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; PRED-NEXT: [[TMP13:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP12]], i32 2
+; PRED-NEXT: store i32 0, ptr [[TMP13]], align 8
+; PRED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; PRED: [[PRED_STORE_CONTINUE2]]:
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; PRED-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IV]], splat (i64 2)
-; PRED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; PRED-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; PRED-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; PRED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; PRED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; PRED: [[MIDDLE_BLOCK]]:
; PRED-NEXT: br label %[[EXIT:.*]]
; PRED: [[SCALAR_PH]]:
@@ -685,5 +685,132 @@ exit:
ret void
}
+define void @exit_cond_zext_iv_store64(ptr %dst, i64 %N) {
+; DEFAULT-LABEL: define void @exit_cond_zext_iv_store64(
+; DEFAULT-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 18
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; DEFAULT: [[VECTOR_SCEVCHECK]]:
+; DEFAULT-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; DEFAULT-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -1
+; DEFAULT-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; DEFAULT-NEXT: [[TMP3:%.*]] = add i32 1, [[TMP2]]
+; DEFAULT-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 1
+; DEFAULT-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; DEFAULT-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; DEFAULT-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 3
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP7]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = trunc i64 [[N_VEC]] to i32
+; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; DEFAULT: [[VECTOR_BODY]]:
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
+; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP9]], i64 2
+; DEFAULT-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP9]], align 8
+; DEFAULT-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP10]], align 8
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; DEFAULT-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; DEFAULT: [[SCALAR_PH]]:
+; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[IV_CONV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IV_CONV]]
+; DEFAULT-NEXT: store i64 0, ptr [[GEP]], align 8
+; DEFAULT-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1
+; DEFAULT-NEXT: [[IV_EXT]] = zext i32 [[IV_1_NEXT]] to i64
+; DEFAULT-NEXT: [[C:%.*]] = icmp ult i64 [[IV_EXT]], [[N]]
+; DEFAULT-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
+; PRED-LABEL: define void @exit_cond_zext_iv_store64(
+; PRED-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
+; PRED-NEXT: [[ENTRY:.*:]]
+; PRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; PRED-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
+; PRED: [[VECTOR_SCEVCHECK]]:
+; PRED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; PRED-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -1
+; PRED-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; PRED-NEXT: [[TMP3:%.*]] = add i32 1, [[TMP2]]
+; PRED-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], 1
+; PRED-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; PRED-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; PRED-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; PRED: [[VECTOR_PH]]:
+; PRED-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 1
+; PRED-NEXT: [[TMP7:%.*]] = and i64 [[N_RND_UP]], 1
+; PRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP7]]
+; PRED-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
+; PRED: [[VECTOR_BODY]]:
+; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; PRED-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; PRED-NEXT: [[TMP8:%.*]] = icmp ule <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PRED-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP8]], i64 0
+; PRED-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; PRED: [[PRED_STORE_IF]]:
+; PRED-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[DST]], i64 [[INDEX]]
+; PRED-NEXT: store i64 0, ptr [[TMP10]], align 8
+; PRED-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; PRED: [[PRED_STORE_CONTINUE]]:
+; PRED-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i64 1
+; PRED-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; PRED: [[PRED_STORE_IF1]]:
+; PRED-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; PRED-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP12]]
+; PRED-NEXT: store i64 0, ptr [[TMP13]], align 8
+; PRED-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; PRED: [[PRED_STORE_CONTINUE2]]:
+; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; PRED-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
+; PRED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; PRED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; PRED: [[MIDDLE_BLOCK]]:
+; PRED-NEXT: br label %[[EXIT:.*]]
+; PRED: [[SCALAR_PH]]:
+; PRED-NEXT: br label %[[LOOP:.*]]
+; PRED: [[LOOP]]:
+; PRED-NEXT: [[IV_1:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
+; PRED-NEXT: [[IV_CONV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_EXT:%.*]], %[[LOOP]] ]
+; PRED-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[DST]], i64 [[IV_CONV]]
+; PRED-NEXT: store i64 0, ptr [[GEP]], align 8
+; PRED-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1
+; PRED-NEXT: [[IV_EXT]] = zext i32 [[IV_1_NEXT]] to i64
+; PRED-NEXT: [[C:%.*]] = icmp ult i64 [[IV_EXT]], [[N]]
+; PRED-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; PRED: [[EXIT]]:
+; PRED-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop ]
+ %iv.conv = phi i64 [ 0, %entry ], [ %iv.ext, %loop ]
+ %gep = getelementptr i64, ptr %dst, i64 %iv.conv
+ store i64 0, ptr %gep, align 8
+ %iv.1.next = add i32 %iv.1, 1
+ %iv.ext = zext i32 %iv.1.next to i64
+ %c = icmp ult i64 %iv.ext, %N
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve" }
More information about the llvm-commits
mailing list