[llvm] [LV] Extend test coverage with branch probabilities (NFC). (PR #209182)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 06:41:51 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/209182
Add tests with branch probabilities, including cost printing tests for replicate regions, VPlan printing tests and masked stores with probabilities.
>From 1e2493703ba09b060764356da347650461d3022a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 30 Jun 2026 12:02:29 +0100
Subject: [PATCH] [LV] Extend test coverage with branch probabilities (NFC).
Add tests with branch probabilities, including cost printing tests for
replicate regions, VPlan printing tests and masked stores with
probabilities.
---
.../AArch64/masked-store-with-profile-info.ll | 63 +++
.../VPlan/vplan-printing-branch-weights.ll | 140 +++++
.../replicate-region-branch-weights.ll | 486 ++++++++++++++++++
.../replicate-region-cost-probability.ll | 195 +++++++
4 files changed, 884 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/masked-store-with-profile-info.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-store-with-profile-info.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-store-with-profile-info.ll
new file mode 100644
index 0000000000000..1907d0ddae146
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-store-with-profile-info.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "masked|br i1|!prof|!llvm.loop|^.*:" --version 6
+; RUN: opt -passes=loop-vectorize -mattr=+sve -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @cond_store(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cond_store(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK: [[VECTOR_BODY:.*]]:
+; CHECK: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[WIDE_LOAD:%.*]], ptr align 4 [[TMP2:%.*]], <4 x i1> [[TMP1:%.*]])
+; CHECK: br i1 [[TMP3:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK: [[LOOP:.*]]:
+; CHECK: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7:![0-9]+]]
+; CHECK: [[IF_THEN]]:
+; CHECK: [[LATCH]]:
+; CHECK: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %val = load i32, ptr %gep.b, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !1
+
+if.then:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %val, ptr %gep.a, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 1000}
+!1 = !{!"branch_weights", i32 1, i32 7}
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 249}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 250}
+; CHECK: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
+; CHECK: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META3]], [[META10:![0-9]+]]}
+; CHECK: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
new file mode 100644
index 0000000000000..da0ef5ec2a96b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-branch-weights.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -vplan-print-after=printAfterInitialConstruction -disable-output %s 2>&1 \
+; RUN: | FileCheck --strict-whitespace %s
+
+define void @predicated_store_profile_metadata(ptr %a, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'predicated_store_profile_metadata'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): loop
+; CHECK-EMPTY:
+; CHECK-NEXT: loop:
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%val> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp sgt ir<%val>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp>
+; CHECK-NEXT: Successor(s): if.then, latch
+; CHECK-EMPTY:
+; CHECK-NEXT: if.then:
+; CHECK-NEXT: EMIT store ir<0>, ir<%gep>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT branch-on-cond ir<%ec>
+; CHECK-NEXT: Successor(s): middle.block, loop
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !1
+
+if.then:
+ store i32 0, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop, !prof !2
+
+exit:
+ ret void
+}
+
+!1 = !{!"branch_weights", i32 1, i32 3}
+!2 = !{!"branch_weights", i32 1, i32 999}
+
+; Two predicated blocks reached with different probabilities each get their own
+; branch_weights, derived from block frequency info. then.1 executes 1 in 4
+; iterations (reciprocal 4, weights {1, 3}); then.2 executes 1 in 8 (reciprocal
+; 8, weights {1, 7}).
+define void @two_predicated_blocks(ptr noalias %a, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'two_predicated_blocks'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): loop
+; CHECK-EMPTY:
+; CHECK-NEXT: loop:
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, latch ]
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%val> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%c1> = icmp sgt ir<%val>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c1>
+; CHECK-NEXT: Successor(s): then.1, if.2
+; CHECK-EMPTY:
+; CHECK-NEXT: then.1:
+; CHECK-NEXT: EMIT store ir<0>, ir<%gep>
+; CHECK-NEXT: Successor(s): if.2
+; CHECK-EMPTY:
+; CHECK-NEXT: if.2:
+; CHECK-NEXT: EMIT ir<%c2> = icmp slt ir<%val>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c2>
+; CHECK-NEXT: Successor(s): then.2, latch
+; CHECK-EMPTY:
+; CHECK-NEXT: then.2:
+; CHECK-NEXT: EMIT store ir<1>, ir<%gep>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT branch-on-cond ir<%ec>
+; CHECK-NEXT: Successor(s): middle.block, loop
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %c1 = icmp sgt i32 %val, 0
+ br i1 %c1, label %then.1, label %if.2, !prof !3
+
+then.1:
+ store i32 0, ptr %gep, align 4
+ br label %if.2
+
+if.2:
+ %c2 = icmp slt i32 %val, 0
+ br i1 %c2, label %then.2, label %latch, !prof !4
+
+then.2:
+ store i32 1, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop, !prof !5
+
+exit:
+ ret void
+}
+
+!3 = !{!"branch_weights", i32 1, i32 3}
+!4 = !{!"branch_weights", i32 1, i32 7}
+!5 = !{!"branch_weights", i32 1, i32 999}
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
new file mode 100644
index 0000000000000..b619eaf1e90fe
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll
@@ -0,0 +1,486 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br i1|!prof|!llvm.loop|^.*:" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=VF4IC1
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -S %s | FileCheck %s --check-prefix=VF2IC2
+
+; Predicated store where the condition is taken 1 in 8 iterations.
+define void @predicated_store(ptr %a, i32 %n) {
+; VF4IC1-LABEL: define void @predicated_store(
+; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF4IC1: [[ENTRY:.*:]]
+; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1: [[VECTOR_BODY:.*]]:
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: [[PRED_STORE_IF]]:
+; VF4IC1: [[PRED_STORE_CONTINUE]]:
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: [[PRED_STORE_IF1]]:
+; VF4IC1: [[PRED_STORE_CONTINUE2]]:
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: [[PRED_STORE_IF3]]:
+; VF4IC1: [[PRED_STORE_CONTINUE4]]:
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: [[PRED_STORE_IF5]]:
+; VF4IC1: [[PRED_STORE_CONTINUE6]]:
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
+; VF4IC1: [[SCALAR_PH]]:
+; VF4IC1: [[LOOP:.*]]:
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7:![0-9]+]]
+; VF4IC1: [[IF_THEN]]:
+; VF4IC1: [[LATCH]]:
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
+; VF4IC1: [[EXIT]]:
+;
+; VF2IC2-LABEL: define void @predicated_store(
+; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF2IC2: [[ENTRY:.*:]]
+; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2: [[VECTOR_BODY:.*]]:
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: [[PRED_STORE_IF2]]:
+; VF2IC2: [[PRED_STORE_CONTINUE3]]:
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: [[PRED_STORE_IF4]]:
+; VF2IC2: [[PRED_STORE_CONTINUE5]]:
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: [[PRED_STORE_IF6]]:
+; VF2IC2: [[PRED_STORE_CONTINUE7]]:
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]
+; VF2IC2: [[SCALAR_PH]]:
+; VF2IC2: [[LOOP:.*]]:
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7:![0-9]+]]
+; VF2IC2: [[IF_THEN]]:
+; VF2IC2: [[LATCH]]:
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
+; VF2IC2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !1
+
+if.then:
+ %add = add i32 %val, 1
+ store i32 %add, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+; Predicated store guarded by %c1 (taken 1/2) and %c2 (taken 1/4), so it
+; executes 1 in 8 iterations.
+define void @predicated_store_sunk_address(ptr %a, ptr %b, i32 %n) {
+; VF4IC1-LABEL: define void @predicated_store_sunk_address(
+; VF4IC1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) {
+; VF4IC1: [[ENTRY:.*:]]
+; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !prof [[PROF0]]
+; VF4IC1: [[VECTOR_MEMCHECK]]:
+; VF4IC1: br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1: [[VECTOR_BODY:.*]]:
+; VF4IC1: br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: [[PRED_STORE_IF]]:
+; VF4IC1: [[PRED_STORE_CONTINUE]]:
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: [[PRED_STORE_IF3]]:
+; VF4IC1: [[PRED_STORE_CONTINUE4]]:
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: [[PRED_STORE_IF5]]:
+; VF4IC1: [[PRED_STORE_CONTINUE6]]:
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; VF4IC1: [[PRED_STORE_IF7]]:
+; VF4IC1: [[PRED_STORE_CONTINUE8]]:
+; VF4IC1: br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: [[SCALAR_PH]]:
+; VF4IC1: [[LOOP:.*]]:
+; VF4IC1: br i1 [[C1:%.*]], label %[[M:.*]], label %[[LATCH:.*]], !prof [[PROF12:![0-9]+]]
+; VF4IC1: [[M]]:
+; VF4IC1: br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF6]]
+; VF4IC1: [[IF]]:
+; VF4IC1: [[LATCH]]:
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP13:![0-9]+]]
+; VF4IC1: [[EXIT]]:
+;
+; VF2IC2-LABEL: define void @predicated_store_sunk_address(
+; VF2IC2-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) {
+; VF2IC2: [[ENTRY:.*:]]
+; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !prof [[PROF0]]
+; VF2IC2: [[VECTOR_MEMCHECK]]:
+; VF2IC2: br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2: [[VECTOR_BODY:.*]]:
+; VF2IC2: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: [[PRED_STORE_IF4]]:
+; VF2IC2: [[PRED_STORE_CONTINUE5]]:
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: [[PRED_STORE_IF6]]:
+; VF2IC2: [[PRED_STORE_CONTINUE7]]:
+; VF2IC2: br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
+; VF2IC2: [[PRED_STORE_IF8]]:
+; VF2IC2: [[PRED_STORE_CONTINUE9]]:
+; VF2IC2: br i1 [[TMP25:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP11:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: [[SCALAR_PH]]:
+; VF2IC2: [[LOOP:.*]]:
+; VF2IC2: br i1 [[C1:%.*]], label %[[M:.*]], label %[[LATCH:.*]], !prof [[PROF12:![0-9]+]]
+; VF2IC2: [[M]]:
+; VF2IC2: br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF6]]
+; VF2IC2: [[IF]]:
+; VF2IC2: [[LATCH]]:
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP13:![0-9]+]]
+; VF2IC2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i32 %iv
+ %v = load i32, ptr %gep.a, align 4
+ %c1 = icmp sgt i32 %v, 0
+ br i1 %c1, label %m, label %latch, !prof !2
+
+m:
+ %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
+ %c2 = icmp sgt i32 %v, 100
+ br i1 %c2, label %if, label %latch, !prof !3
+
+if:
+ store i32 %v, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+; Loop without profile data.
+define void @predicated_store_no_profile(ptr %a, i32 %n) {
+; VF4IC1-LABEL: define void @predicated_store_no_profile(
+; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF4IC1: [[ENTRY:.*:]]
+; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1: [[VECTOR_BODY:.*]]:
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: [[PRED_STORE_IF]]:
+; VF4IC1: [[PRED_STORE_CONTINUE]]:
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: [[PRED_STORE_IF1]]:
+; VF4IC1: [[PRED_STORE_CONTINUE2]]:
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: [[PRED_STORE_IF3]]:
+; VF4IC1: [[PRED_STORE_CONTINUE4]]:
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: [[PRED_STORE_IF5]]:
+; VF4IC1: [[PRED_STORE_CONTINUE6]]:
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VF4IC1: [[SCALAR_PH]]:
+; VF4IC1: [[LOOP:.*]]:
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
+; VF4IC1: [[IF_THEN]]:
+; VF4IC1: [[LATCH]]:
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; VF4IC1: [[EXIT]]:
+;
+; VF2IC2-LABEL: define void @predicated_store_no_profile(
+; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF2IC2: [[ENTRY:.*:]]
+; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2: [[VECTOR_BODY:.*]]:
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: [[PRED_STORE_IF2]]:
+; VF2IC2: [[PRED_STORE_CONTINUE3]]:
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: [[PRED_STORE_IF4]]:
+; VF2IC2: [[PRED_STORE_CONTINUE5]]:
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: [[PRED_STORE_IF6]]:
+; VF2IC2: [[PRED_STORE_CONTINUE7]]:
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; VF2IC2: [[SCALAR_PH]]:
+; VF2IC2: [[LOOP:.*]]:
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
+; VF2IC2: [[IF_THEN]]:
+; VF2IC2: [[LATCH]]:
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; VF2IC2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch
+
+if.then:
+ %add = add i32 %val, 1
+ store i32 %add, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Profile data for the loop latch, but not internal loop blocks.
+define void @predicated_store_unprofiled_predicate(ptr %a, i32 %n) {
+; VF4IC1-LABEL: define void @predicated_store_unprofiled_predicate(
+; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF4IC1: [[ENTRY:.*:]]
+; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1: [[VECTOR_BODY:.*]]:
+; VF4IC1: br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: [[PRED_STORE_IF]]:
+; VF4IC1: [[PRED_STORE_CONTINUE]]:
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: [[PRED_STORE_IF1]]:
+; VF4IC1: [[PRED_STORE_CONTINUE2]]:
+; VF4IC1: br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: [[PRED_STORE_IF3]]:
+; VF4IC1: [[PRED_STORE_CONTINUE4]]:
+; VF4IC1: br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: [[PRED_STORE_IF5]]:
+; VF4IC1: [[PRED_STORE_CONTINUE6]]:
+; VF4IC1: br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: [[SCALAR_PH]]:
+; VF4IC1: [[LOOP:.*]]:
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
+; VF4IC1: [[IF_THEN]]:
+; VF4IC1: [[LATCH]]:
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP17:![0-9]+]]
+; VF4IC1: [[EXIT]]:
+;
+; VF2IC2-LABEL: define void @predicated_store_unprofiled_predicate(
+; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF2IC2: [[ENTRY:.*:]]
+; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2: [[VECTOR_BODY:.*]]:
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: [[PRED_STORE_IF2]]:
+; VF2IC2: [[PRED_STORE_CONTINUE3]]:
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: [[PRED_STORE_IF4]]:
+; VF2IC2: [[PRED_STORE_CONTINUE5]]:
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: [[PRED_STORE_IF6]]:
+; VF2IC2: [[PRED_STORE_CONTINUE7]]:
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: [[SCALAR_PH]]:
+; VF2IC2: [[LOOP:.*]]:
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
+; VF2IC2: [[IF_THEN]]:
+; VF2IC2: [[LATCH]]:
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP17:![0-9]+]]
+; VF2IC2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch
+
+if.then:
+ %add = add i32 %val, 1
+ store i32 %add, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+define void @predicated_store_almost_always_taken(ptr %a, i32 %n) {
+; VF4IC1-LABEL: define void @predicated_store_almost_always_taken(
+; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF4IC1: [[ENTRY:.*:]]
+; VF4IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1: [[VECTOR_BODY:.*]]:
+; VF4IC1: br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4IC1: [[PRED_STORE_IF]]:
+; VF4IC1: [[PRED_STORE_CONTINUE]]:
+; VF4IC1: br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4IC1: [[PRED_STORE_IF1]]:
+; VF4IC1: [[PRED_STORE_CONTINUE2]]:
+; VF4IC1: br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4IC1: [[PRED_STORE_IF3]]:
+; VF4IC1: [[PRED_STORE_CONTINUE4]]:
+; VF4IC1: br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; VF4IC1: [[PRED_STORE_IF5]]:
+; VF4IC1: [[PRED_STORE_CONTINUE6]]:
+; VF4IC1: br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF4IC1: [[SCALAR_PH]]:
+; VF4IC1: [[LOOP:.*]]:
+; VF4IC1: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF19:![0-9]+]]
+; VF4IC1: [[IF_THEN]]:
+; VF4IC1: [[LATCH]]:
+; VF4IC1: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP20:![0-9]+]]
+; VF4IC1: [[EXIT]]:
+;
+; VF2IC2-LABEL: define void @predicated_store_almost_always_taken(
+; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
+; VF2IC2: [[ENTRY:.*:]]
+; VF2IC2: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2: [[VECTOR_BODY:.*]]:
+; VF2IC2: br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2: br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC2: [[PRED_STORE_IF2]]:
+; VF2IC2: [[PRED_STORE_CONTINUE3]]:
+; VF2IC2: br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; VF2IC2: [[PRED_STORE_IF4]]:
+; VF2IC2: [[PRED_STORE_CONTINUE5]]:
+; VF2IC2: br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
+; VF2IC2: [[PRED_STORE_IF6]]:
+; VF2IC2: [[PRED_STORE_CONTINUE7]]:
+; VF2IC2: br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2: br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF6]]
+; VF2IC2: [[SCALAR_PH]]:
+; VF2IC2: [[LOOP:.*]]:
+; VF2IC2: br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF19:![0-9]+]]
+; VF2IC2: [[IF_THEN]]:
+; VF2IC2: [[LATCH]]:
+; VF2IC2: br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP20:![0-9]+]]
+; VF2IC2: [[EXIT]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !4
+
+if.then:
+ %add = add i32 %val, 1
+ store i32 %add, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 1000}
+!1 = !{!"branch_weights", i32 1, i32 7}
+!2 = !{!"branch_weights", i32 1, i32 1}
+!3 = !{!"branch_weights", i32 1, i32 3}
+!4 = !{!"branch_weights", i32 1000, i32 1}
+;.
+; VF4IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; VF4IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 249}
+; VF4IC1: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
+; VF4IC1: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; VF4IC1: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VF4IC1: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 250}
+; VF4IC1: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
+; VF4IC1: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
+; VF4IC1: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
+; VF4IC1: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META3]], [[META10:![0-9]+]]}
+; VF4IC1: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+; VF4IC1: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]], [[META5]]}
+; VF4IC1: [[PROF12]] = !{!"branch_weights", i32 1, i32 1}
+; VF4IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META10]]}
+; VF4IC1: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]], [[META4]]}
+; VF4IC1: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]], [[META3]]}
+; VF4IC1: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]], [[META4]], [[META5]]}
+; VF4IC1: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]], [[META3]], [[META10]]}
+; VF4IC1: [[LOOP18]] = distinct !{[[LOOP18]], [[META3]], [[META4]], [[META5]]}
+; VF4IC1: [[PROF19]] = !{!"branch_weights", i32 1000, i32 1}
+; VF4IC1: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]], [[META3]], [[META10]]}
+;.
+; VF2IC2: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; VF2IC2: [[PROF1]] = !{!"branch_weights", i32 1, i32 249}
+; VF2IC2: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
+; VF2IC2: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; VF2IC2: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VF2IC2: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 250}
+; VF2IC2: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
+; VF2IC2: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
+; VF2IC2: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
+; VF2IC2: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META3]], [[META10:![0-9]+]]}
+; VF2IC2: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
+; VF2IC2: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]], [[META5]]}
+; VF2IC2: [[PROF12]] = !{!"branch_weights", i32 1, i32 1}
+; VF2IC2: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META10]]}
+; VF2IC2: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]], [[META4]]}
+; VF2IC2: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]], [[META3]]}
+; VF2IC2: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]], [[META4]], [[META5]]}
+; VF2IC2: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]], [[META3]], [[META10]]}
+; VF2IC2: [[LOOP18]] = distinct !{[[LOOP18]], [[META3]], [[META4]], [[META5]]}
+; VF2IC2: [[PROF19]] = !{!"branch_weights", i32 1000, i32 1}
+; VF2IC2: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]], [[META3]], [[META10]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll b/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
new file mode 100644
index 0000000000000..1928c2c3e995a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*REPLICATE" --version 6
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -disable-output -debug-only=loop-vectorize %s 2>&1 | FileCheck %s
+
+; The predicate is taken 1 in 4 iterations (reciprocal 4): cost 10 / 4 -> 2.5.
+define void @predicated_udiv_p25(ptr %a, i32 %n) {
+; CHECK-LABEL: 'predicated_udiv_p25'
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6]]>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !1
+
+if.then:
+ %d = udiv i32 %val, 7
+ store i32 %d, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+; The predicate is taken 1 in 8 iterations (reciprocal 8): cost 10 / 8 -> 1.25.
+; A less likely predicate makes the predicated udiv look cheaper than at 1/4.
+define void @predicated_udiv_p12(ptr %a, i32 %n) {
+; CHECK-LABEL: 'predicated_udiv_p12'
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
+; CHECK: Cost of 1.25 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6]]>
+; CHECK: Cost of 1.25 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !2
+
+if.then:
+ %d = udiv i32 %val, 7
+ store i32 %d, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+; Without profile data the cost model is still self-contained: the recorded
+; reciprocal comes from block frequency info, which defaults to an even 50/50
+; split for the unprofiled predicate (reciprocal 2), so cost 10 / 2 -> 5.
+define void @predicated_udiv_no_profile(ptr %a, i32 %n) {
+; CHECK-LABEL: 'predicated_udiv_no_profile'
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
+; CHECK: Cost of 5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6]]>
+; CHECK: Cost of 5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch
+
+if.then:
+ %d = udiv i32 %val, 7
+ store i32 %d, ptr %gep, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two adjacent predicated blocks guarded by the same condition are merged into a
+; single replicate region. 2 branches with same condition have different probabilities.
+define void @merged_regions_use_conservative_probability(ptr noalias %a, ptr noalias %b, i32 %n, i32 %x, i32 %y) {
+; CHECK-LABEL: 'merged_regions_use_conservative_probability'
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7:%[0-9]+]]>
+; CHECK: Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6:%[0-9]+]]>, ir<%gep>.1
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
+; CHECK: Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7]]>
+; CHECK: Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6]]>, ir<%gep>.1
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
+; CHECK: Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %then1, label %bb1, !prof !1
+
+then1:
+ %d1 = sdiv i32 %val, %x
+ store i32 %d1, ptr %gep, align 4
+ br label %bb1
+
+bb1:
+ br i1 %cmp, label %then2, label %latch, !prof !3
+
+then2:
+ %d2 = sdiv i32 %val, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
+ store i32 %d2, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+define void @sunk_recipe_uses_region_probability(ptr noalias %a, ptr noalias %b, i32 %n, i32 %x) {
+; CHECK-LABEL: 'sunk_recipe_uses_region_probability'
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%rem> = srem ir<%val>, ir<%x>
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP6:%[0-9]+]]>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%rem>, ir<%gep.b>
+; CHECK: Cost of 2.5 for VF 2: REPLICATE ir<%rem> = srem ir<%val>, ir<%x>
+; CHECK: Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP6]]>
+; CHECK: Cost of 0 for VF 2: REPLICATE store ir<%rem>, ir<%gep.b>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %val = load i32, ptr %gep, align 4
+ %cmp = icmp sgt i32 %val, 0
+ br i1 %cmp, label %if.then, label %latch, !prof !1
+
+if.then:
+ %rem = srem i32 %val, %x
+ %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
+ store i32 %rem, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop, !prof !0
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 1000}
+!1 = !{!"branch_weights", i32 1, i32 3}
+!2 = !{!"branch_weights", i32 1, i32 7}
+!3 = !{!"branch_weights", i32 1, i32 99}
More information about the llvm-commits
mailing list