[llvm] [LV] Add tests with branch weights and estimated trip counts (NFC) (PR #215978)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 01:00:10 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/215978
Extend test coverage for branch weight handling, including tests for currently missing cases, including missing branch weights in the middle blocks and epilogue loops.
>From 33523b5d5dacbbbab5df4711a24e14fb8e4f3606 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 4 Aug 2026 08:27:13 +0100
Subject: [PATCH] [LV] Add tests with branch weights and estimated trip counts
(NFC)
Extend test coverage for branch weight handling, including tests for
currently missing cases, including missing branch weights in the middle
blocks and epilogue loops.
---
.../AArch64/branch-weights-scalar-epilogue.ll | 85 ++++
.../LoopVectorize/AArch64/check-prof-info.ll | 20 +-
.../branch-weights-scalable-vf1.ll | 58 +++
.../LoopVectorize/branch-weights.ll | 463 ++++++++++++++++++
.../LoopVectorize/check-prof-info.ll | 190 ++++---
5 files changed, 729 insertions(+), 87 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/branch-weights-scalar-epilogue.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/branch-weights-scalable-vf1.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/branch-weights-scalar-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/branch-weights-scalar-epilogue.ll
new file mode 100644
index 0000000000000..c930ef5db147d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/branch-weights-scalar-epilogue.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br " --filter "^.*:" --filter "icmp" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -S %s | FileCheck %s
+
+; The interleave group of the loads has a gap, so the epilogue vector loop
+; requires a scalar epilogue and its minimum iteration check leaves it for a
+; remainder of exactly the epilogue's step of 4. Check the branch weights of
+; that check for an estimated remainder of exactly 4 iterations, that is, for the
+; boundary at which the epilogue vector loop is estimated to be skipped.
+define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) !prof !0 {
+; CHECK-LABEL: define i32 @load_factor_4_with_tail_gap(
+; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]]) #[[ATTR0:[0-9]+]] !prof [[PROF0:![0-9]+]] {
+; CHECK: [[ITER_CHECK:.*:]]
+; CHECK: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4
+; CHECK: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], [[TMP1:%.*]]
+; CHECK: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF1]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF:%.*]], 0
+; CHECK: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK: [[TMP45:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; CHECK: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5:%.*]], 4
+; CHECK: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK: [[TMP48:%.*]] = icmp eq i64 [[TMP47:%.*]], 0
+; CHECK: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK: [[TMP55:%.*]] = icmp eq i64 [[INDEX_NEXT20:%.*]], [[N_VEC13:%.*]]
+; CHECK: br i1 [[TMP55]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK: br label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; CHECK: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
+ %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0
+ %ld1 = load i32, ptr %arrayidx, align 4
+ %add = add nsw i32 %rdx, %ld1
+ %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1
+ %ld2 = load i32, ptr %arrayidx1, align 4
+ %add1 = add nsw i32 %add, %ld2
+ %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 2
+ %ld3 = load i32, ptr %arrayidx2, align 4
+ %rdx.next = add nsw i32 %add1, %ld3
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop, !prof !1
+
+exit:
+ ret i32 %rdx.next
+}
+
+!0 = !{!"function_entry_count", i64 13}
+; The estimated trip count is 1028, so vectorizing by the main loop's step of 16
+; leaves a remainder of 4 iterations.
+!1 = !{!"branch_weights", i32 1, i32 1027}
+;.
+; CHECK: [[PROF0]] = !{!"function_entry_count", i64 13}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 1, i32 63}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 64}
+; CHECK: [[PROF7]] = !{!"branch_weights", i32 4, i32 12}
+; CHECK: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META5]], [[META10:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]], [[META4]], [[META12:![0-9]+]]}
+; CHECK: [[META12]] = !{!"llvm.loop.estimated_trip_count", i32 0}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
index 5d4182108c2e1..106d046a8e2d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
@@ -38,7 +38,7 @@ define void @foo_i32(i64 %n) {
; CHECK-V1-IC1-FORCE-EPI2: [[VECTOR_PH]]:
; CHECK-V1-IC1-FORCE-EPI2: br label %[[VECTOR_BODY:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VECTOR_BODY]]:
-; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP5:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[MIDDLE_BLOCK]]:
; CHECK-V1-IC1-FORCE-EPI2: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF5:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_ITER_CHECK]]:
@@ -46,7 +46,7 @@ define void @foo_i32(i64 %n) {
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_PH]]:
; CHECK-V1-IC1-FORCE-EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP8:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-V1-IC1-FORCE-EPI2: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF9:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_SCALAR_PH]]:
@@ -124,7 +124,7 @@ define void @foo_i8(i64 %n) {
; CHECK-V1-IC1: [[VECTOR_PH]]:
; CHECK-V1-IC1: br label %[[VECTOR_BODY:.*]]
; CHECK-V1-IC1: [[VECTOR_BODY]]:
-; CHECK-V1-IC1: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-V1-IC1: br i1 [[TMP5:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-V1-IC1: [[MIDDLE_BLOCK]]:
; CHECK-V1-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF9]]
; CHECK-V1-IC1: [[VEC_EPILOG_ITER_CHECK]]:
@@ -132,7 +132,7 @@ define void @foo_i8(i64 %n) {
; CHECK-V1-IC1: [[VEC_EPILOG_PH]]:
; CHECK-V1-IC1: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-V1-IC1: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-V1-IC1: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-V1-IC1: br i1 [[TMP8:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-V1-IC1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-V1-IC1: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF14:![0-9]+]]
; CHECK-V1-IC1: [[VEC_EPILOG_SCALAR_PH]]:
@@ -150,7 +150,7 @@ define void @foo_i8(i64 %n) {
; CHECK-V1-IC1-FORCE-EPI2: [[VECTOR_PH]]:
; CHECK-V1-IC1-FORCE-EPI2: br label %[[VECTOR_BODY:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VECTOR_BODY]]:
-; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF12:![0-9]+]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP5:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF12:![0-9]+]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[MIDDLE_BLOCK]]:
; CHECK-V1-IC1-FORCE-EPI2: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF12]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_ITER_CHECK]]:
@@ -158,7 +158,7 @@ define void @foo_i8(i64 %n) {
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_PH]]:
; CHECK-V1-IC1-FORCE-EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP8:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-V1-IC1-FORCE-EPI2: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF9]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_SCALAR_PH]]:
@@ -176,7 +176,7 @@ define void @foo_i8(i64 %n) {
; CHECK-V2-IC1: [[VECTOR_PH]]:
; CHECK-V2-IC1: br label %[[VECTOR_BODY:.*]]
; CHECK-V2-IC1: [[VECTOR_BODY]]:
-; CHECK-V2-IC1: br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF10:![0-9]+]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-V2-IC1: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF10:![0-9]+]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-V2-IC1: [[MIDDLE_BLOCK]]:
; CHECK-V2-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF13:![0-9]+]]
; CHECK-V2-IC1: [[VEC_EPILOG_ITER_CHECK]]:
@@ -184,7 +184,7 @@ define void @foo_i8(i64 %n) {
; CHECK-V2-IC1: [[VEC_EPILOG_PH]]:
; CHECK-V2-IC1: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-V2-IC1: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-V2-IC1: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-V2-IC1: br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-V2-IC1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-V2-IC1: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF6]]
; CHECK-V2-IC1: [[VEC_EPILOG_SCALAR_PH]]:
@@ -262,7 +262,7 @@ define void @foo_i32_no_bw(i64 %n) {
; CHECK-V1-IC1-FORCE-EPI2: [[VECTOR_PH]]:
; CHECK-V1-IC1-FORCE-EPI2: br label %[[VECTOR_BODY:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VECTOR_BODY]]:
-; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP5:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[MIDDLE_BLOCK]]:
; CHECK-V1-IC1-FORCE-EPI2: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_ITER_CHECK]]:
@@ -270,7 +270,7 @@ define void @foo_i32_no_bw(i64 %n) {
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_PH]]:
; CHECK-V1-IC1-FORCE-EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI2: br i1 [[TMP8:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-V1-IC1-FORCE-EPI2: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-V1-IC1-FORCE-EPI2: [[VEC_EPILOG_SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/branch-weights-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/branch-weights-scalable-vf1.ll
new file mode 100644
index 0000000000000..7a4e9f61698ba
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/branch-weights-scalable-vf1.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br " --filter "^.*:" --filter "icmp" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width="vscale x 1" -force-vector-interleave=1 -S %s | FileCheck %s
+
+; Without a vscale for tuning, a VF of vscale x 1 leaves an estimated vector step
+; of 1, for which every estimated trip count is a multiple of the step. Check
+; that neither successor of the middle block's terminator ends up with a weight
+; of zero, which would give the loop the terminator leaves a profile count of 0.
+define void @vector_step_of_one(ptr %p, i64 %n) !prof !0 {
+; CHECK-LABEL: define void @vector_step_of_one(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0:![0-9]+]] {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP0:%.*]]
+; CHECK: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; CHECK: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; CHECK: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF7]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ store i32 0, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !1
+
+exit:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 13}
+; The estimated trip count is 1024.
+!1 = !{!"branch_weights", i32 1, i32 1023}
+;.
+; CHECK: [[PROF0]] = !{!"function_entry_count", i64 13}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 1, i32 1023}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 1024}
+; CHECK: [[PROF7]] = !{!"branch_weights", i32 1, i32 0}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]], [[META4]], [[META9:![0-9]+]]}
+; CHECK: [[META9]] = !{!"llvm.loop.estimated_trip_count", i32 0}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/branch-weights.ll b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
index 6a272105f2887..e8e95bccf0413 100644
--- a/llvm/test/Transforms/LoopVectorize/branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
@@ -3,6 +3,8 @@
; RUN: -epilogue-vectorization-force-VF=4 | FileCheck %s --check-prefix=MAINVF4IC1_EPI4
; RUN: opt < %s -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -enable-epilogue-vectorization \
; RUN: -epilogue-vectorization-force-VF=4 | FileCheck %s --check-prefix=MAINVF4IC2_EPI4
+; RUN: opt < %s -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -enable-epilogue-vectorization \
+; RUN: -epilogue-vectorization-force-VF=2 | FileCheck %s --check-prefix=MAINVF8IC1_EPI2
define void @f0(i8 %n, i32 %len, ptr %p) !prof !0 {
; MAINVF4IC1_EPI4-LABEL: define void @f0(
@@ -91,6 +93,48 @@ define void @f0(i8 %n, i32 %len, ptr %p) !prof !0 {
; MAINVF4IC2_EPI4: br label %[[EXIT]]
; MAINVF4IC2_EPI4: [[EXIT]]:
;
+; MAINVF8IC1_EPI2-LABEL: define void @f0(
+; MAINVF8IC1_EPI2-SAME: i8 [[N:%.*]], i32 [[LEN:%.*]], ptr [[P:%.*]]) !prof [[PROF0:![0-9]+]] {
+; MAINVF8IC1_EPI2: [[ENTRY:.*:]]
+; MAINVF8IC1_EPI2: [[CMP_ENTRY:%.*]] = icmp sgt i32 [[LEN]], 0
+; MAINVF8IC1_EPI2: br i1 [[CMP_ENTRY]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]], !prof [[PROF1:![0-9]+]]
+; MAINVF8IC1_EPI2: [[ITER_CHECK]]:
+; MAINVF8IC1_EPI2: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]], !prof [[PROF2:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VECTOR_SCEVCHECK]]:
+; MAINVF8IC1_EPI2: [[TMP2:%.*]] = icmp slt i8 [[TMP1:%.*]], 0
+; MAINVF8IC1_EPI2: [[TMP3:%.*]] = icmp ugt i32 [[LEN]], 255
+; MAINVF8IC1_EPI2: br i1 [[TMP4:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0:%.*]], 8
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF3:![0-9]+]], !llvm.loop [[LOOP4:![0-9]+]]
+; MAINVF8IC1_EPI2: [[MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF:%.*]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT5:%.*]], [[N_VEC2:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF10:![0-9]+]], !llvm.loop [[LOOP11:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N7:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC2]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N7]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[LOOP:.*]]
+; MAINVF8IC1_EPI2: [[LOOP]]:
+; MAINVF8IC1_EPI2: [[CMP_LOOP:%.*]] = icmp ult i32 [[I32:%.*]], [[LEN]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_LOOP]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !prof [[PROF14:![0-9]+]], !llvm.loop [[LOOP15:![0-9]+]]
+; MAINVF8IC1_EPI2: [[EXIT_LOOPEXIT]]:
+; MAINVF8IC1_EPI2: br label %[[EXIT]]
+; MAINVF8IC1_EPI2: [[EXIT]]:
+;
entry:
%cmp.entry = icmp sgt i32 %len, 0
br i1 %cmp.entry, label %loop, label %exit, !prof !1
@@ -112,9 +156,371 @@ exit:
ret void
}
+define void @f1(ptr %p, i64 %n) !prof !0 {
+; MAINVF4IC1_EPI4-LABEL: define void @f1(
+; MAINVF4IC1_EPI4-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF4IC1_EPI4: [[ITER_CHECK:.*:]]
+; MAINVF4IC1_EPI4: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAINVF4IC1_EPI4: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC1_EPI4: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF4IC1_EPI4: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4: [[VECTOR_PH]]:
+; MAINVF4IC1_EPI4: br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4: [[VECTOR_BODY]]:
+; MAINVF4IC1_EPI4: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC1_EPI4: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF17:![0-9]+]], !llvm.loop [[LOOP18:![0-9]+]]
+; MAINVF4IC1_EPI4: [[MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC1_EPI4: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF:%.*]], 4
+; MAINVF4IC1_EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_PH]]:
+; MAINVF4IC1_EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC1_EPI4: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT5:%.*]], [[N_VEC3:%.*]]
+; MAINVF4IC1_EPI4: br i1 [[TMP3]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF10]], !llvm.loop [[LOOP19:![0-9]+]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_N6]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC1_EPI4: br label %[[LOOP:.*]]
+; MAINVF4IC1_EPI4: [[LOOP]]:
+; MAINVF4IC1_EPI4: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF17]], !llvm.loop [[LOOP21:![0-9]+]]
+; MAINVF4IC1_EPI4: [[EXIT]]:
+;
+; MAINVF4IC2_EPI4-LABEL: define void @f1(
+; MAINVF4IC2_EPI4-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF4IC2_EPI4: [[ITER_CHECK:.*:]]
+; MAINVF4IC2_EPI4: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAINVF4IC2_EPI4: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC2_EPI4: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF4IC2_EPI4: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4: [[VECTOR_PH]]:
+; MAINVF4IC2_EPI4: br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4: [[VECTOR_BODY]]:
+; MAINVF4IC2_EPI4: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC2_EPI4: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF17:![0-9]+]], !llvm.loop [[LOOP18:![0-9]+]]
+; MAINVF4IC2_EPI4: [[MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC2_EPI4: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF:%.*]], 4
+; MAINVF4IC2_EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_PH]]:
+; MAINVF4IC2_EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC2_EPI4: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5:%.*]], [[N_VEC3:%.*]]
+; MAINVF4IC2_EPI4: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF10]], !llvm.loop [[LOOP19:![0-9]+]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_N6]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC2_EPI4: br label %[[LOOP:.*]]
+; MAINVF4IC2_EPI4: [[LOOP]]:
+; MAINVF4IC2_EPI4: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF17]], !llvm.loop [[LOOP21:![0-9]+]]
+; MAINVF4IC2_EPI4: [[EXIT]]:
+;
+; MAINVF8IC1_EPI2-LABEL: define void @f1(
+; MAINVF8IC1_EPI2-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF8IC1_EPI2: [[ITER_CHECK:.*:]]
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF16:![0-9]+]], !llvm.loop [[LOOP17:![0-9]+]]
+; MAINVF8IC1_EPI2: [[MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF:%.*]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT5:%.*]], [[N_VEC3:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP3]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF16]], !llvm.loop [[LOOP19:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N6]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[LOOP:.*]]
+; MAINVF8IC1_EPI2: [[LOOP]]:
+; MAINVF8IC1_EPI2: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF10]], !llvm.loop [[LOOP20:![0-9]+]]
+; MAINVF8IC1_EPI2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ store i32 0, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !3
+
+exit:
+ ret void
+}
+
+define void @f2(ptr %p, i64 %n) !prof !0 {
+; MAINVF4IC1_EPI4-LABEL: define void @f2(
+; MAINVF4IC1_EPI4-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF4IC1_EPI4: [[ITER_CHECK:.*:]]
+; MAINVF4IC1_EPI4: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAINVF4IC1_EPI4: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC1_EPI4: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF4IC1_EPI4: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4: [[VECTOR_PH]]:
+; MAINVF4IC1_EPI4: br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4: [[VECTOR_BODY]]:
+; MAINVF4IC1_EPI4: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC1_EPI4: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP22:![0-9]+]]
+; MAINVF4IC1_EPI4: [[MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC1_EPI4: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF:%.*]], 4
+; MAINVF4IC1_EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_PH]]:
+; MAINVF4IC1_EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC1_EPI4: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT5:%.*]], [[N_VEC3:%.*]]
+; MAINVF4IC1_EPI4: br i1 [[TMP3]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_N6]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC1_EPI4: br label %[[LOOP:.*]]
+; MAINVF4IC1_EPI4: [[LOOP]]:
+; MAINVF4IC1_EPI4: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF10]], !llvm.loop [[LOOP25:![0-9]+]]
+; MAINVF4IC1_EPI4: [[EXIT]]:
+;
+; MAINVF4IC2_EPI4-LABEL: define void @f2(
+; MAINVF4IC2_EPI4-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF4IC2_EPI4: [[ITER_CHECK:.*:]]
+; MAINVF4IC2_EPI4: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAINVF4IC2_EPI4: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC2_EPI4: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF4IC2_EPI4: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4: [[VECTOR_PH]]:
+; MAINVF4IC2_EPI4: br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4: [[VECTOR_BODY]]:
+; MAINVF4IC2_EPI4: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC2_EPI4: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP22:![0-9]+]]
+; MAINVF4IC2_EPI4: [[MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC2_EPI4: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF:%.*]], 4
+; MAINVF4IC2_EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_PH]]:
+; MAINVF4IC2_EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC2_EPI4: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT5:%.*]], [[N_VEC3:%.*]]
+; MAINVF4IC2_EPI4: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_N6]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC2_EPI4: br label %[[LOOP:.*]]
+; MAINVF4IC2_EPI4: [[LOOP]]:
+; MAINVF4IC2_EPI4: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF10]], !llvm.loop [[LOOP25:![0-9]+]]
+; MAINVF4IC2_EPI4: [[EXIT]]:
+;
+; MAINVF8IC1_EPI2-LABEL: define void @f2(
+; MAINVF8IC1_EPI2-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF8IC1_EPI2: [[ITER_CHECK:.*:]]
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP21:![0-9]+]]
+; MAINVF8IC1_EPI2: [[MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF:%.*]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT5:%.*]], [[N_VEC3:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP3]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N6]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[LOOP:.*]]
+; MAINVF8IC1_EPI2: [[LOOP]]:
+; MAINVF8IC1_EPI2: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF10]], !llvm.loop [[LOOP25:![0-9]+]]
+; MAINVF8IC1_EPI2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ store i32 0, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !4
+
+exit:
+ ret void
+}
+
+define void @f3(ptr %p, i64 %n) !prof !0 {
+; MAINVF4IC1_EPI4-LABEL: define void @f3(
+; MAINVF4IC1_EPI4-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF4IC1_EPI4: [[ITER_CHECK:.*:]]
+; MAINVF4IC1_EPI4: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAINVF4IC1_EPI4: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC1_EPI4: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF4IC1_EPI4: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4: [[VECTOR_PH]]:
+; MAINVF4IC1_EPI4: br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4: [[VECTOR_BODY]]:
+; MAINVF4IC1_EPI4: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC1_EPI4: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; MAINVF4IC1_EPI4: [[MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC1_EPI4: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0:%.*]], 4
+; MAINVF4IC1_EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_PH]]:
+; MAINVF4IC1_EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC1_EPI4: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT4:%.*]], [[N_VEC2:%.*]]
+; MAINVF4IC1_EPI4: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4: [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC2]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_N5]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF4IC1_EPI4: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC1_EPI4: br label %[[LOOP:.*]]
+; MAINVF4IC1_EPI4: [[LOOP]]:
+; MAINVF4IC1_EPI4: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF4IC1_EPI4: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF28:![0-9]+]], !llvm.loop [[LOOP29:![0-9]+]]
+; MAINVF4IC1_EPI4: [[EXIT]]:
+;
+; MAINVF4IC2_EPI4-LABEL: define void @f3(
+; MAINVF4IC2_EPI4-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF4IC2_EPI4: [[ITER_CHECK:.*:]]
+; MAINVF4IC2_EPI4: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; MAINVF4IC2_EPI4: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC2_EPI4: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF4IC2_EPI4: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4: [[VECTOR_PH]]:
+; MAINVF4IC2_EPI4: br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4: [[VECTOR_BODY]]:
+; MAINVF4IC2_EPI4: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC2_EPI4: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; MAINVF4IC2_EPI4: [[MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC2_EPI4: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0:%.*]], 4
+; MAINVF4IC2_EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_PH]]:
+; MAINVF4IC2_EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC2_EPI4: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT4:%.*]], [[N_VEC2:%.*]]
+; MAINVF4IC2_EPI4: br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4: [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC2]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_N5]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF4IC2_EPI4: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC2_EPI4: br label %[[LOOP:.*]]
+; MAINVF4IC2_EPI4: [[LOOP]]:
+; MAINVF4IC2_EPI4: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF4IC2_EPI4: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF28:![0-9]+]], !llvm.loop [[LOOP29:![0-9]+]]
+; MAINVF4IC2_EPI4: [[EXIT]]:
+;
+; MAINVF8IC1_EPI2-LABEL: define void @f3(
+; MAINVF8IC1_EPI2-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) !prof [[PROF0]] {
+; MAINVF8IC1_EPI2: [[ITER_CHECK:.*:]]
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; MAINVF8IC1_EPI2: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF8IC1_EPI2: [[VECTOR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; MAINVF8IC1_EPI2: [[MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF8]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF8IC1_EPI2: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0:%.*]], 2
+; MAINVF8IC1_EPI2: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_PH]]:
+; MAINVF8IC1_EPI2: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF8IC1_EPI2: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT4:%.*]], [[N_VEC2:%.*]]
+; MAINVF8IC1_EPI2: br i1 [[TMP5]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF8IC1_EPI2: [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC2]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_N5]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF13]]
+; MAINVF8IC1_EPI2: [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF8IC1_EPI2: br label %[[LOOP:.*]]
+; MAINVF8IC1_EPI2: [[LOOP]]:
+; MAINVF8IC1_EPI2: [[CMP_LOOP:%.*]] = icmp eq i64 [[IV_NEXT:%.*]], [[N]]
+; MAINVF8IC1_EPI2: br i1 [[CMP_LOOP]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF28:![0-9]+]], !llvm.loop [[LOOP29:![0-9]+]]
+; MAINVF8IC1_EPI2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptr = getelementptr inbounds i32, ptr %p, i64 %iv
+ store i32 0, ptr %ptr
+ %iv.next = add i64 %iv, 1
+ %cmp.loop = icmp eq i64 %iv.next, %n
+ br i1 %cmp.loop, label %exit, label %loop, !prof !4, !llvm.loop !5
+
+exit:
+ ret void
+}
+
!0 = !{!"function_entry_count", i64 13}
!1 = !{!"branch_weights", i32 12, i32 1}
!2 = !{!"branch_weights", i32 1234, i32 1}
+; The estimated trip count is 31, so vectorizing by 8 leaves a remainder of 7.
+!3 = !{!"branch_weights", i32 1, i32 30}
+; The estimated trip count is 1024, so vectorizing by 8 leaves no remainder.
+!4 = !{!"branch_weights", i32 1, i32 1023}
+; An explicit estimated trip count of 0 takes precedence over the latch weights,
+; which on their own would estimate a trip count of 1024.
+!5 = distinct !{!5, !6}
+!6 = !{!"llvm.loop.estimated_trip_count", i32 0}
;.
; MAINVF4IC1_EPI4: [[PROF0]] = !{!"function_entry_count", i64 13}
; MAINVF4IC1_EPI4: [[PROF1]] = !{!"branch_weights", i32 12, i32 1}
@@ -133,6 +539,19 @@ exit:
; MAINVF4IC1_EPI4: [[PROF14]] = !{!"branch_weights", i32 2, i32 1}
; MAINVF4IC1_EPI4: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META16:![0-9]+]]}
; MAINVF4IC1_EPI4: [[META16]] = !{!"llvm.loop.estimated_trip_count", i32 3}
+; MAINVF4IC1_EPI4: [[PROF17]] = !{!"branch_weights", i32 1, i32 2}
+; MAINVF4IC1_EPI4: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]], [[META6]], [[META16]]}
+; MAINVF4IC1_EPI4: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]], [[META6]], [[META20:![0-9]+]]}
+; MAINVF4IC1_EPI4: [[META20]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+; MAINVF4IC1_EPI4: [[LOOP21]] = distinct !{[[LOOP21]], [[META6]], [[META5]], [[META16]]}
+; MAINVF4IC1_EPI4: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]], [[META6]], [[META23:![0-9]+]]}
+; MAINVF4IC1_EPI4: [[META23]] = !{!"llvm.loop.estimated_trip_count", i32 128}
+; MAINVF4IC1_EPI4: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]], [[META12]], [[META6]]}
+; MAINVF4IC1_EPI4: [[LOOP25]] = distinct !{[[LOOP25]], [[META6]], [[META5]], [[META12]]}
+; MAINVF4IC1_EPI4: [[LOOP26]] = distinct !{[[LOOP26]], [[META12]], [[META5]], [[META6]]}
+; MAINVF4IC1_EPI4: [[LOOP27]] = distinct !{[[LOOP27]], [[META12]], [[META5]], [[META6]]}
+; MAINVF4IC1_EPI4: [[PROF28]] = !{!"branch_weights", i32 1, i32 1023}
+; MAINVF4IC1_EPI4: [[LOOP29]] = distinct !{[[LOOP29]], [[META12]], [[META6]], [[META5]]}
;.
; MAINVF4IC2_EPI4: [[PROF0]] = !{!"function_entry_count", i64 13}
; MAINVF4IC2_EPI4: [[PROF1]] = !{!"branch_weights", i32 12, i32 1}
@@ -151,4 +570,48 @@ exit:
; MAINVF4IC2_EPI4: [[PROF14]] = !{!"branch_weights", i32 2, i32 1}
; MAINVF4IC2_EPI4: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META16:![0-9]+]]}
; MAINVF4IC2_EPI4: [[META16]] = !{!"llvm.loop.estimated_trip_count", i32 3}
+; MAINVF4IC2_EPI4: [[PROF17]] = !{!"branch_weights", i32 1, i32 2}
+; MAINVF4IC2_EPI4: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]], [[META6]], [[META16]]}
+; MAINVF4IC2_EPI4: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]], [[META6]], [[META20:![0-9]+]]}
+; MAINVF4IC2_EPI4: [[META20]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+; MAINVF4IC2_EPI4: [[LOOP21]] = distinct !{[[LOOP21]], [[META6]], [[META5]], [[META16]]}
+; MAINVF4IC2_EPI4: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]], [[META6]], [[META23:![0-9]+]]}
+; MAINVF4IC2_EPI4: [[META23]] = !{!"llvm.loop.estimated_trip_count", i32 128}
+; MAINVF4IC2_EPI4: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]], [[META12]], [[META6]]}
+; MAINVF4IC2_EPI4: [[LOOP25]] = distinct !{[[LOOP25]], [[META6]], [[META5]], [[META12]]}
+; MAINVF4IC2_EPI4: [[LOOP26]] = distinct !{[[LOOP26]], [[META12]], [[META5]], [[META6]]}
+; MAINVF4IC2_EPI4: [[LOOP27]] = distinct !{[[LOOP27]], [[META12]], [[META5]], [[META6]]}
+; MAINVF4IC2_EPI4: [[PROF28]] = !{!"branch_weights", i32 1, i32 1023}
+; MAINVF4IC2_EPI4: [[LOOP29]] = distinct !{[[LOOP29]], [[META12]], [[META6]], [[META5]]}
+;.
+; MAINVF8IC1_EPI2: [[PROF0]] = !{!"function_entry_count", i64 13}
+; MAINVF8IC1_EPI2: [[PROF1]] = !{!"branch_weights", i32 12, i32 1}
+; MAINVF8IC1_EPI2: [[PROF2]] = !{!"branch_weights", i32 1, i32 127}
+; MAINVF8IC1_EPI2: [[PROF3]] = !{!"branch_weights", i32 1, i32 153}
+; MAINVF8IC1_EPI2: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; MAINVF8IC1_EPI2: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAINVF8IC1_EPI2: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAINVF8IC1_EPI2: [[META7]] = !{!"llvm.loop.estimated_trip_count", i32 154}
+; MAINVF8IC1_EPI2: [[PROF8]] = !{!"branch_weights", i32 1, i32 7}
+; MAINVF8IC1_EPI2: [[PROF9]] = !{!"branch_weights", i32 2, i32 6}
+; MAINVF8IC1_EPI2: [[PROF10]] = !{!"branch_weights", i32 1, i32 0}
+; MAINVF8IC1_EPI2: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]], [[META6]], [[META12:![0-9]+]]}
+; MAINVF8IC1_EPI2: [[META12]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+; MAINVF8IC1_EPI2: [[PROF13]] = !{!"branch_weights", i32 1, i32 1}
+; MAINVF8IC1_EPI2: [[PROF14]] = !{!"branch_weights", i32 0, i32 1}
+; MAINVF8IC1_EPI2: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META12]]}
+; MAINVF8IC1_EPI2: [[PROF16]] = !{!"branch_weights", i32 1, i32 2}
+; MAINVF8IC1_EPI2: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META6]], [[META18:![0-9]+]]}
+; MAINVF8IC1_EPI2: [[META18]] = !{!"llvm.loop.estimated_trip_count", i32 3}
+; MAINVF8IC1_EPI2: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]], [[META6]], [[META18]]}
+; MAINVF8IC1_EPI2: [[LOOP20]] = distinct !{[[LOOP20]], [[META6]], [[META5]], [[META12]]}
+; MAINVF8IC1_EPI2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]], [[META6]], [[META22:![0-9]+]]}
+; MAINVF8IC1_EPI2: [[META22]] = !{!"llvm.loop.estimated_trip_count", i32 128}
+; MAINVF8IC1_EPI2: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]], [[META24:![0-9]+]], [[META6]]}
+; MAINVF8IC1_EPI2: [[META24]] = !{!"llvm.loop.estimated_trip_count", i32 0}
+; MAINVF8IC1_EPI2: [[LOOP25]] = distinct !{[[LOOP25]], [[META6]], [[META5]], [[META24]]}
+; MAINVF8IC1_EPI2: [[LOOP26]] = distinct !{[[LOOP26]], [[META24]], [[META5]], [[META6]]}
+; MAINVF8IC1_EPI2: [[LOOP27]] = distinct !{[[LOOP27]], [[META24]], [[META5]], [[META6]]}
+; MAINVF8IC1_EPI2: [[PROF28]] = !{!"branch_weights", i32 1, i32 1023}
+; MAINVF8IC1_EPI2: [[LOOP29]] = distinct !{[[LOOP29]], [[META24]], [[META6]], [[META5]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/check-prof-info.ll b/llvm/test/Transforms/LoopVectorize/check-prof-info.ll
index fbe276ff6c853..fbbfda563e74e 100644
--- a/llvm/test/Transforms/LoopVectorize/check-prof-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/check-prof-info.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:"
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:" --version 6
; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s -check-prefix=CHECK-MASKED
; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 \
@@ -11,45 +11,45 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; Check correctness of profile info for vectorization without epilog.
define void @_Z3foov() {
-; CHECK-LABEL: @_Z3foov(
-; CHECK: entry:
-; CHECK: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK: br i1 [[TMP4:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF0:![0-9]+]], !llvm.loop [[LOOP1:![0-9]+]]
-; CHECK: middle.block:
-; CHECK: br label [[FOR_BODY:%.*]]
-; CHECK: for.cond.cleanup:
+; CHECK-LABEL: define void @_Z3foov() {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK: br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0:![0-9]+]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK: [[FOR_COND_CLEANUP]]:
;
-; CHECK-MASKED-LABEL: @_Z3foov(
-; CHECK-MASKED: entry:
-; CHECK-MASKED: br label [[VECTOR_PH:%.*]]
-; CHECK-MASKED: vector.ph:
-; CHECK-MASKED: br label [[VECTOR_BODY:%.*]]
-; CHECK-MASKED: vector.body:
-; CHECK-MASKED: br i1 [[TMP16:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF0:![0-9]+]], !llvm.loop [[LOOP1:![0-9]+]]
-; CHECK-MASKED: middle.block:
-; CHECK-MASKED: br label [[FOR_BODY:%.*]]
-; CHECK-MASKED: for.cond.cleanup:
+; CHECK-MASKED-LABEL: define void @_Z3foov() {
+; CHECK-MASKED: [[ENTRY:.*:]]
+; CHECK-MASKED: br label %[[VECTOR_PH:.*]]
+; CHECK-MASKED: [[VECTOR_PH]]:
+; CHECK-MASKED: br label %[[VECTOR_BODY:.*]]
+; CHECK-MASKED: [[VECTOR_BODY]]:
+; CHECK-MASKED: br i1 [[TMP16:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0:![0-9]+]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-MASKED: [[MIDDLE_BLOCK]]:
+; CHECK-MASKED: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-MASKED: [[FOR_COND_CLEANUP]]:
;
-; CHECK-SCALABLE-LABEL: @_Z3foov(
-; CHECK-SCALABLE: entry:
-; CHECK-SCALABLE: br i1 [[MIN_ITERS_CHECK:%.*]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
-; CHECK-SCALABLE: vector.ph:
+; CHECK-SCALABLE-LABEL: define void @_Z3foov() {
+; CHECK-SCALABLE: [[ENTRY:.*:]]
+; CHECK-SCALABLE: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-SCALABLE: [[VECTOR_PH]]:
; CHECK-SCALABLE: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9:%.*]], i64 0
; CHECK-SCALABLE: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-SCALABLE: br label [[VECTOR_BODY:%.*]]
-; CHECK-SCALABLE: vector.body:
+; CHECK-SCALABLE: br label %[[VECTOR_BODY:.*]]
+; CHECK-SCALABLE: [[VECTOR_BODY]]:
; CHECK-SCALABLE: [[VEC_IND_NEXT:%.*]] = add <vscale x 4 x i32> [[VEC_IND:%.*]], [[BROADCAST_SPLAT]]
-; CHECK-SCALABLE: br i1 [[TMP11:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
-; CHECK-SCALABLE: middle.block:
-; CHECK-SCALABLE: br i1 [[CMP_N:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
-; CHECK-SCALABLE: scalar.ph:
-; CHECK-SCALABLE: br label [[FOR_BODY:%.*]]
-; CHECK-SCALABLE: for.cond.cleanup:
-; CHECK-SCALABLE: for.body:
-; CHECK-SCALABLE: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF7:![0-9]+]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-SCALABLE: br i1 [[TMP10:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-SCALABLE: [[MIDDLE_BLOCK]]:
+; CHECK-SCALABLE: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK-SCALABLE: [[SCALAR_PH]]:
+; CHECK-SCALABLE: br label %[[FOR_BODY:.*]]
+; CHECK-SCALABLE: [[FOR_COND_CLEANUP]]:
+; CHECK-SCALABLE: [[FOR_BODY]]:
+; CHECK-SCALABLE: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF7:![0-9]+]], !llvm.loop [[LOOP8:![0-9]+]]
;
entry:
br label %for.body
@@ -74,53 +74,53 @@ for.body:
; Check correctness of profile info for vectorization with epilog.
define void @_Z3foo2v() {
-; CHECK-LABEL: @_Z3foo2v(
-; CHECK: entry:
-; CHECK: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK: br i1 [[TMP4:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: middle.block:
-; CHECK: br label [[SCALAR_PH:%.*]]
-; CHECK: scalar.ph:
-; CHECK: br label [[FOR_BODY:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK: for.body:
-; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-LABEL: define void @_Z3foo2v() {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK: br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK: [[FOR_BODY]]:
+; CHECK: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
;
-; CHECK-MASKED-LABEL: @_Z3foo2v(
-; CHECK-MASKED: entry:
-; CHECK-MASKED: br label [[VECTOR_PH:%.*]]
-; CHECK-MASKED: vector.ph:
-; CHECK-MASKED: br label [[VECTOR_BODY:%.*]]
-; CHECK-MASKED: vector.body:
-; CHECK-MASKED: br i1 [[TMP16:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK-MASKED: middle.block:
-; CHECK-MASKED: br label [[SCALAR_PH:%.*]]
-; CHECK-MASKED: scalar.ph:
-; CHECK-MASKED: br label [[FOR_BODY:%.*]]
-; CHECK-MASKED: for.cond.cleanup:
-; CHECK-MASKED: for.body:
-; CHECK-MASKED: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-MASKED-LABEL: define void @_Z3foo2v() {
+; CHECK-MASKED: [[ENTRY:.*:]]
+; CHECK-MASKED: br label %[[VECTOR_PH:.*]]
+; CHECK-MASKED: [[VECTOR_PH]]:
+; CHECK-MASKED: br label %[[VECTOR_BODY:.*]]
+; CHECK-MASKED: [[VECTOR_BODY]]:
+; CHECK-MASKED: br i1 [[TMP16:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-MASKED: [[MIDDLE_BLOCK]]:
+; CHECK-MASKED: br label %[[SCALAR_PH:.*]]
+; CHECK-MASKED: [[SCALAR_PH]]:
+; CHECK-MASKED: br label %[[FOR_BODY:.*]]
+; CHECK-MASKED: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-MASKED: [[FOR_BODY]]:
+; CHECK-MASKED: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
;
-; CHECK-SCALABLE-LABEL: @_Z3foo2v(
-; CHECK-SCALABLE: entry:
-; CHECK-SCALABLE: br i1 [[MIN_ITERS_CHECK:%.*]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
-; CHECK-SCALABLE: vector.ph:
+; CHECK-SCALABLE-LABEL: define void @_Z3foo2v() {
+; CHECK-SCALABLE: [[ENTRY:.*:]]
+; CHECK-SCALABLE: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-SCALABLE: [[VECTOR_PH]]:
; CHECK-SCALABLE: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9:%.*]], i64 0
; CHECK-SCALABLE: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-SCALABLE: br label [[VECTOR_BODY:%.*]]
-; CHECK-SCALABLE: vector.body:
+; CHECK-SCALABLE: br label %[[VECTOR_BODY:.*]]
+; CHECK-SCALABLE: [[VECTOR_BODY]]:
; CHECK-SCALABLE: [[VEC_IND_NEXT:%.*]] = add <vscale x 4 x i32> [[VEC_IND:%.*]], [[BROADCAST_SPLAT]]
-; CHECK-SCALABLE: br i1 [[TMP11:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK-SCALABLE: middle.block:
-; CHECK-SCALABLE: br i1 [[CMP_N:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF6]]
-; CHECK-SCALABLE: scalar.ph:
-; CHECK-SCALABLE: br label [[FOR_BODY:%.*]]
-; CHECK-SCALABLE: for.cond.cleanup:
-; CHECK-SCALABLE: for.body:
-; CHECK-SCALABLE: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF11:![0-9]+]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-SCALABLE: br i1 [[TMP10:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-SCALABLE: [[MIDDLE_BLOCK]]:
+; CHECK-SCALABLE: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; CHECK-SCALABLE: [[SCALAR_PH]]:
+; CHECK-SCALABLE: br label %[[FOR_BODY:.*]]
+; CHECK-SCALABLE: [[FOR_COND_CLEANUP]]:
+; CHECK-SCALABLE: [[FOR_BODY]]:
+; CHECK-SCALABLE: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF11:![0-9]+]], !llvm.loop [[LOOP12:![0-9]+]]
;
entry:
br label %for.body
@@ -146,3 +146,39 @@ for.body:
!0 = !{!"branch_weights", i32 1, i32 1023}
!1 = !{!"branch_weights", i32 1, i32 1026}
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 1, i32 255}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META4]] = !{!"llvm.loop.estimated_trip_count", i32 256}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META3]], [[META4]]}
+; CHECK: [[PROF6]] = !{!"branch_weights", i32 1, i32 2}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]], [[META2]], [[META8:![0-9]+]]}
+; CHECK: [[META8]] = !{!"llvm.loop.estimated_trip_count", i32 3}
+;.
+; CHECK-MASKED: [[PROF0]] = !{!"branch_weights", i32 1, i32 63}
+; CHECK-MASKED: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK-MASKED: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-MASKED: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-MASKED: [[META4]] = !{!"llvm.loop.estimated_trip_count", i32 64}
+; CHECK-MASKED: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META3]], [[META4]]}
+; CHECK-MASKED: [[PROF6]] = !{!"branch_weights", i32 1, i32 2}
+; CHECK-MASKED: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]], [[META2]], [[META8:![0-9]+]]}
+; CHECK-MASKED: [[META8]] = !{!"llvm.loop.estimated_trip_count", i32 3}
+;.
+; CHECK-SCALABLE: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK-SCALABLE: [[PROF1]] = !{!"branch_weights", i32 1, i32 255}
+; CHECK-SCALABLE: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
+; CHECK-SCALABLE: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-SCALABLE: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-SCALABLE: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 256}
+; CHECK-SCALABLE: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-SCALABLE: [[PROF7]] = !{!"branch_weights", i32 1, i32 0}
+; CHECK-SCALABLE: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META3]], [[META9:![0-9]+]]}
+; CHECK-SCALABLE: [[META9]] = !{!"llvm.loop.estimated_trip_count", i32 0}
+; CHECK-SCALABLE: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META4]], [[META5]]}
+; CHECK-SCALABLE: [[PROF11]] = !{!"branch_weights", i32 1, i32 2}
+; CHECK-SCALABLE: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META3]], [[META13:![0-9]+]]}
+; CHECK-SCALABLE: [[META13]] = !{!"llvm.loop.estimated_trip_count", i32 3}
+;.
More information about the llvm-commits
mailing list