[llvm-branch-commits] [llvm] [NFC][VPlan] Add tests for partial linearization (PR #217484)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 28 16:52:15 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/217484
>From 517bb9ee9453a445417e5066afe61167f59aab16 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 14 Aug 2026 12:20:20 -0700
Subject: [PATCH 1/4] Predicator tests for uniform control flow preservation
---
.../LoopVectorize/VPlan/predicator.ll | 168 ++++++++++++++++++
1 file changed, 168 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 86597a10bd7b8..e47b68ed39ce9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -949,3 +949,171 @@ latch:
exit:
ret void
}
+
+; @outermost_uniform_branch
+; bb0 (u)
+; / \
+; bb1 (v) \
+; / | |
+; bb2 | |
+; \ | |
+; bb3 [phi] |
+; \ /
+; bb4 [phi]
+;
+; bb0's uniform branch can be easily preserved.
+
+; @outermost_uniform_branch_more_blocks
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 [phi] |
+; \ /
+; bb6 [phi]
+;
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+
+; @uniform_branch_after_varying_branch
+; bb0 (v)
+; / \
+; bb4 bb1 (u)
+; | / |
+; | bb2 |
+; | \ |
+; | bb3 [phi]
+; \ /
+; bb5 [phi]
+
+; @uniform_branch_after_varying_branch_more_blocks
+; bb0 (v)
+; / \
+; bb5 bb1 (u)
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4 [phi]
+; \ /
+; bb6 [phi]
+;
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+
+; @uniform_branch_after_varying_branch_more_blocks_mirrored
+; bb0 (v)
+; / \
+; bb2 (u) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 [phi] |
+; \ /
+; bb6 [phi]
+;
+; Mirror of the above test so that "(u)" block would be processed before/after
+; the other "(v)" destination by the RPOT.
+
+; @uniform_branch_on_masked_def
+; bb0 (v)
+; / \
+; | bb1 (br i1 (load i1 %uni.ptr))
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4
+; \ /
+; bb5 [phi]
+;
+; bb1's condition is uniform, but its value is loaded only along one path of
+; bb0's varying branch. It cannot be preserved trivially, because some the load
+; may never be executed.
+
+
+; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform
+; control flow merging into the middle of the varying diamond, but "covering"
+; different potential RPOT traversals.
+
+; @uniform_branch_unstructured_merge1
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; bb4 bb3 [phi]
+; \ /
+; bb5 [phi]
+;
+
+; @uniform_branch_unstructured_merge2
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; | +-\---+
+; | / \
+; bb4 [phi] bb3
+; \ /
+; bb5 [phi]
+;
+
+; @uniform_branch_unstructured_merge3
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; bb4 [phi] bb2
+; \ /
+; bb5 [phi]
+;
+
+; @uniform_branch_unstructured_merge4
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; +-/--+ \
+; / \ \
+; bb4 bb2 [phi]
+; \ /
+; bb5 [phi]
+
+
+
+; @uniform_branch_shared_join_with_varying
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / | |
+; bb3 | /
+; \ | /
+; bb4 [phi]
+;
+; Theoretically can be done, but would require a combination of a blend and a phi.
+
+; @unstructured_uniform_only
+; bb0 (u)
+; / \
+; bb2 (u) bb1
+; / \ /
+; bb4 bb3 [phi]
+; \ /
+; bb5 [phi]
+;
+; Triviallly preservable, but detection might not be easy.
+
+; @unstructured_uniform_only_sese_region
+; bb0 (v)
+; / \
+; bb6 bb1 (u)
+; | / \
+; | bb3 (u) bb2
+; | / \ /
+; | bb5 bb4 [phi]
+; | \ /
+; | bb7 [phi]
+; \ /
+; \ /
+; bb8 [phi]
+;
>From 141aef50058366165d2e0370fba60078ecee1c97 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 14 Aug 2026 14:23:21 -0700
Subject: [PATCH 2/4] Create actual test functions (AI-assisted)
---
.../LoopVectorize/VPlan/predicator.ll | 503 +++++++++++++++++-
1 file changed, 478 insertions(+), 25 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index e47b68ed39ce9..b2b65fe185968 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -950,7 +950,11 @@ exit:
ret void
}
-; @outermost_uniform_branch
+define void @outermost_uniform_branch(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (u)
; / \
; bb1 (v) \
@@ -960,10 +964,41 @@ exit:
; bb3 [phi] |
; \ /
; bb4 [phi]
-;
; bb0's uniform branch can be easily preserved.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ %add0 = add i64 %iv, 0
+ br i1 %u0, label %bb1, label %bb4
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add3, %bb3 ], [ %add0, %bb0 ]
+ store i64 %phi4, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
-; @outermost_uniform_branch_more_blocks
+bb0:
; bb0 (u)
; / \
; bb2 (v) bb1
@@ -973,11 +1008,48 @@ exit:
; bb5 [phi] |
; \ /
; bb6 [phi]
-;
; Similar to above, but with extra blocks on some edges. Probably doesn't
; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb6
+
+bb2:
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
-; @uniform_branch_after_varying_branch
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ %add5 = add i64 %phi5, 5
+ br label %bb6
+
+bb6:
+ %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ]
+ store i64 %phi6, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (v)
; / \
; bb4 bb1 (u)
@@ -987,8 +1059,43 @@ exit:
; | bb3 [phi]
; \ /
; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb4, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br i1 %u1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb3
-; @uniform_branch_after_varying_branch_more_blocks
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (v)
; / \
; bb5 bb1 (u)
@@ -998,11 +1105,48 @@ exit:
; | bb4 [phi]
; \ /
; bb6 [phi]
-;
; Similar to above, but with extra blocks on some edges. Probably doesn't
; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb6
+
+bb5:
+ %add5 = add i64 %iv, 5
+ br label %bb6
+
+bb6:
+ %phi6 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ]
+ store i64 %phi6, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) {
+entry:
+ br label %bb0
-; @uniform_branch_after_varying_branch_more_blocks_mirrored
+bb0:
; bb0 (v)
; / \
; bb2 (u) bb1
@@ -1012,11 +1156,48 @@ exit:
; bb5 [phi] |
; \ /
; bb6 [phi]
-;
; Mirror of the above test so that "(u)" block would be processed before/after
; the other "(v)" destination by the RPOT.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb6
+
+bb2:
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ %add5 = add i64 %phi5, 5
+ br label %bb6
-; @uniform_branch_on_masked_def
+bb6:
+ %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ]
+ store i64 %phi6, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (v)
; / \
; | bb1 (br i1 (load i1 %uni.ptr))
@@ -1026,17 +1207,50 @@ exit:
; | bb4
; \ /
; bb5 [phi]
-;
; bb1's condition is uniform, but its value is loaded only along one path of
; bb0's varying branch. It cannot be preserved trivially, because some the load
; may never be executed.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %add0 = add i64 %iv, 0
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+bb1:
+ %cu1 = load i1, ptr %uni.ptr1
+ br i1 %cu1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add0, %bb0 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform
; control flow merging into the middle of the varying diamond, but "covering"
; different potential RPOT traversals.
-
-; @uniform_branch_unstructured_merge1
; bb0 (u)
; / \
; bb2 (v) bb1
@@ -1044,9 +1258,43 @@ exit:
; bb4 bb3 [phi]
; \ /
; bb5 [phi]
-;
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
-; @uniform_branch_unstructured_merge2
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge2(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (u)
; / \
; bb2 (v) bb1
@@ -1056,9 +1304,43 @@ exit:
; bb4 [phi] bb3
; \ /
; bb5 [phi]
-;
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb5
+
+bb4:
+ %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
-; @uniform_branch_unstructured_merge3
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (u)
; / \
; bb3 bb1 (v)
@@ -1066,9 +1348,43 @@ exit:
; bb4 [phi] bb2
; \ /
; bb5 [phi]
-;
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
-; @uniform_branch_unstructured_merge4
+bb5:
+ %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (u)
; / \
; bb3 bb1 (v)
@@ -1078,10 +1394,43 @@ exit:
; bb4 bb2 [phi]
; \ /
; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ %phi2 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ]
+ %add2 = add i64 %phi2, 2
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb2
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+exit:
+ ret void
+}
+define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
-; @uniform_branch_shared_join_with_varying
+bb0:
; bb0 (u)
; / \
; bb2 (v) bb1
@@ -1089,10 +1438,39 @@ exit:
; bb3 | /
; \ | /
; bb4 [phi]
-;
; Theoretically can be done, but would require a combination of a blend and a phi.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb3, label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ], [ %add3, %bb3 ]
+ store i64 %phi4, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) {
+entry:
+ br label %bb0
-; @unstructured_uniform_only
+bb0:
; bb0 (u)
; / \
; bb2 (u) bb1
@@ -1100,10 +1478,43 @@ exit:
; bb4 bb3 [phi]
; \ /
; bb5 [phi]
-;
; Triviallly preservable, but detection might not be easy.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
-; @unstructured_uniform_only_sese_region
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) {
+entry:
+ br label %bb0
+
+bb0:
; bb0 (v)
; / \
; bb6 bb1 (u)
@@ -1116,4 +1527,46 @@ exit:
; \ /
; \ /
; bb8 [phi]
-;
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb8 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb6, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br i1 %u3, label %bb5, label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb7
+
+bb5:
+ %add5 = add i64 %iv, 5
+ br label %bb7
+
+bb6:
+ %add6 = add i64 %iv, 6
+ br label %bb8
+
+bb7:
+ %phi7 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ]
+ %add7 = add i64 %phi7, 7
+ br label %bb8
+
+bb8:
+ %phi8 = phi i64 [ %add6, %bb6 ], [ %add7, %bb7 ]
+ store i64 %phi8, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
>From ac4758ad4f5c0e8c04bf6e46c69e2e08afbcae95 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 18 Aug 2026 11:24:57 -0700
Subject: [PATCH 3/4] Add `*_no_phi` versions
---
.../LoopVectorize/VPlan/predicator.ll | 649 +++++++++++++++++-
1 file changed, 636 insertions(+), 13 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index b2b65fe185968..ec95d78b7b7c0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -950,6 +950,7 @@ exit:
ret void
}
+;; Uniform control flow preservation tests
define void @outermost_uniform_branch(ptr %a, i1 %u0) {
entry:
br label %bb0
@@ -994,6 +995,50 @@ exit:
ret void
}
+define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb1 (v) \
+; / | |
+; bb2 | |
+; \ | |
+; bb3 |
+; \ /
+; bb4
+; bb0's uniform branch can be easily preserved.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %add0 = add i64 %iv, 0
+ store i64 %add0, ptr %gep
+ br i1 %u0, label %bb1, label %bb4
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb3
+
+bb3:
+ br label %bb4
+
+bb4:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
entry:
br label %bb0
@@ -1045,6 +1090,57 @@ exit:
ret void
}
+define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 |
+; \ /
+; bb6
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb6
+
+bb2:
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ br label %bb6
+
+bb6:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
entry:
br label %bb0
@@ -1091,6 +1187,52 @@ exit:
ret void
}
+define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb4 bb1 (u)
+; | / |
+; | bb2 |
+; | \ |
+; | bb3
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb4, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br i1 %u1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb3
+
+bb3:
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
entry:
br label %bb0
@@ -1142,6 +1284,57 @@ exit:
ret void
}
+define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %u1) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb5 bb1 (u)
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4
+; \ /
+; bb6
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ br label %bb6
+
+bb5:
+ %add5 = add i64 %iv, 5
+ store i64 %add5, ptr %gep
+ br label %bb6
+
+bb6:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) {
entry:
br label %bb0
@@ -1193,6 +1386,57 @@ exit:
ret void
}
+define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr %a, i1 %u2) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb2 (u) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 |
+; \ /
+; bb6
+; Mirror of the above test so that "(u)" block would be processed before/after
+; the other "(v)" destination by the RPOT.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb6
+
+bb2:
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ br label %bb6
+
+bb6:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) {
entry:
br label %bb0
@@ -1221,19 +1465,116 @@ bb1:
bb2:
%add2 = add i64 %iv, 2
- br label %bb4
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add0, %bb0 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_on_masked_def_no_phi(ptr %a, ptr %uni.ptr1) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; | bb1 (br i1 (load i1 %uni.ptr))
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4
+; \ /
+; bb5
+; bb1's condition is uniform, but its value is loaded only along one path of
+; bb0's varying branch. It cannot be preserved trivially, because some the load
+; may never be executed.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %add0 = add i64 %iv, 0
+ store i64 %add0, ptr %gep
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ %cu1 = load i1, ptr %uni.ptr1
+ br i1 %cu1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform
+; control flow merging into the middle of the varying diamond, but "covering"
+; different potential RPOT traversals.
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; bb4 bb3 [phi]
+; \ /
+; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
bb3:
- %add3 = add i64 %iv, 3
- br label %bb4
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
bb4:
- %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
- %add4 = add i64 %phi4, 4
+ %add4 = add i64 %iv, 4
br label %bb5
bb5:
- %phi5 = phi i64 [ %add0, %bb0 ], [ %add4, %bb4 ]
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
store i64 %phi5, ptr %a
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
@@ -1243,7 +1584,7 @@ exit:
ret void
}
-define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) {
+define void @uniform_branch_unstructured_merge1_no_phi(ptr %a, i1 %u0) {
entry:
br label %bb0
@@ -1255,33 +1596,33 @@ bb0:
; / \
; bb2 (v) bb1
; / \ /
-; bb4 bb3 [phi]
+; bb4 bb3
; \ /
-; bb5 [phi]
+; bb5
%iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
br i1 %u0, label %bb2, label %bb1
bb1:
%add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
br label %bb3
bb2:
%add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
%v2 = icmp sle i64 %iv, 2
br i1 %v2, label %bb4, label %bb3
bb3:
- %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
- %add3 = add i64 %phi3, 3
br label %bb5
bb4:
%add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
br label %bb5
bb5:
- %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
- store i64 %phi5, ptr %a
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %bb0
@@ -1336,6 +1677,52 @@ exit:
ret void
}
+define void @uniform_branch_unstructured_merge2_no_phi(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; | +-\---+
+; | / \
+; bb4 bb3
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb5
+
+bb4:
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) {
entry:
br label %bb0
@@ -1380,6 +1767,50 @@ exit:
ret void
}
+define void @uniform_branch_unstructured_merge3_no_phi(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; bb4 bb2
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) {
entry:
br label %bb0
@@ -1426,6 +1857,52 @@ exit:
ret void
}
+define void @uniform_branch_unstructured_merge4_no_phi(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; +-/--+ \
+; / \ \
+; bb4 bb2
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb2
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
entry:
br label %bb0
@@ -1466,6 +1943,48 @@ exit:
ret void
}
+define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / | /
+; bb3 | /
+; \ | /
+; bb4
+; Theoretically can be done, but would require a combination of a blend and a phi.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb3, label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) {
entry:
br label %bb0
@@ -1510,6 +2029,50 @@ exit:
ret void
}
+define void @unstructured_uniform_only_no_phi(ptr %a, i1 %u0, i1 %u2) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (u) bb1
+; / \ /
+; bb4 bb3
+; \ /
+; bb5
+; Triviallly preservable, but detection might not be easy.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) {
entry:
br label %bb0
@@ -1570,3 +2133,63 @@ bb8:
exit:
ret void
}
+
+define void @unstructured_uniform_only_sese_region_no_phi(ptr %a, i1 %u1, i1 %u3) {
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb6 bb1 (u)
+; | / \
+; | bb3 (u) bb2
+; | / \ /
+; | bb5 bb4
+; | \ /
+; | bb7
+; \ /
+; \ /
+; bb8
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb8 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb6, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br i1 %u3, label %bb5, label %bb4
+
+bb4:
+ br label %bb7
+
+bb5:
+ %add5 = add i64 %iv, 5
+ store i64 %add5, ptr %gep
+ br label %bb7
+
+bb6:
+ %add6 = add i64 %iv, 6
+ store i64 %add6, ptr %gep
+ br label %bb8
+
+bb7:
+ br label %bb8
+
+bb8:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
>From 5373f2dcd56db7b1099ce2a1c9bb9c80ced52295 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 12:16:07 -0700
Subject: [PATCH 4/4] Copy to LoopVectorize/predicator.ll and generate CHECKs
in both
---
.../LoopVectorize/VPlan/predicator.ll | 1127 ++++++++-
.../Transforms/LoopVectorize/predicator.ll | 2165 +++++++++++++++++
2 files changed, 3275 insertions(+), 17 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index ec95d78b7b7c0..4b4f8c4b035f0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -disable-output < %s -p loop-vectorize -vplan-print-after=introduceMasksAndLinearize -vplan-print-vector-region-scope 2>&1 | FileCheck %s
+; RUN: opt -disable-output < %s -p loop-vectorize -vplan-print-after=introduceMasksAndLinearize -vplan-print-vector-region-scope -vplan-print-metadata=false 2>&1 | FileCheck %s
define void @diamond_phi(ptr %a) {
; CHECK-LABEL: VPlan for loop in 'diamond_phi'
@@ -952,6 +952,40 @@ exit:
;; Uniform control flow preservation tests
define void @outermost_uniform_branch(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%v1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> ir<%add3>/ir<%u0>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -996,6 +1030,40 @@ exit:
}
define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%iv>, ir<%gep>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1040,6 +1108,50 @@ exit:
}
define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch_more_blocks'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%v2>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: BLEND ir<%phi6> = ir<%add1>/vp<[[VP4]]> ir<%add5>/ir<%u0>
+; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1091,6 +1203,50 @@ exit:
}
define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch_more_blocks_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1142,6 +1298,45 @@ exit:
}
define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%u1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP4]]> ir<%add4>/ir<%v0>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1188,6 +1383,45 @@ exit:
}
define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1234,6 +1468,50 @@ exit:
}
define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_more_blocks'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%u1>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%iv>, ir<5>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: BLEND ir<%phi6> = ir<%add4>/vp<[[VP4]]> ir<%add5>/ir<%v0>
+; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1285,6 +1563,50 @@ exit:
}
define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %u1) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_more_blocks_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%iv>, ir<5>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add5>, ir<%gep>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1336,22 +1658,66 @@ exit:
}
define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) {
-entry:
- br label %bb0
-
-bb0:
-; bb0 (v)
-; / \
-; bb2 (u) bb1
-; / \ |
-; bb4 bb3 |
-; \ / |
-; bb5 [phi] |
-; \ /
-; bb6 [phi]
-; Mirror of the above test so that "(u)" block would be processed before/after
-; the other "(v)" destination by the RPOT.
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_more_blocks_mirrored'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%u2>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: BLEND ir<%phi6> = ir<%add1>/vp<[[VP4]]> ir<%add5>/ir<%v0>
+; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb2 (u) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 [phi] |
+; \ /
+; bb6 [phi]
+; Mirror of the above test so that "(u)" block would be processed before/after
+; the other "(v)" destination by the RPOT.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
%v0 = icmp sle i64 %iv, 0
br i1 %v0, label %bb2, label %bb1
@@ -1387,6 +1753,50 @@ exit:
}
define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr %a, i1 %u2) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1438,6 +1848,47 @@ exit:
}
define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_on_masked_def'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: EMIT-SCALAR ir<%cu1> = load ir<%uni.ptr1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%cu1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cu1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%cu1>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%iv>/ir<true> ir<%add4>/vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1488,6 +1939,47 @@ exit:
}
define void @uniform_branch_on_masked_def_no_phi(ptr %a, ptr %uni.ptr1) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_on_masked_def_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%iv>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: EMIT-SCALAR ir<%cu1> = load ir<%uni.ptr1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%cu1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cu1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1538,6 +2030,48 @@ exit:
}
define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge1'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP5]]> ir<%add3>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1585,6 +2119,48 @@ exit:
}
define void @uniform_branch_unstructured_merge1_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge1_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1632,6 +2208,48 @@ exit:
}
define void @uniform_branch_unstructured_merge2(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge2'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP6]]> ir<%add4>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1678,6 +2296,48 @@ exit:
}
define void @uniform_branch_unstructured_merge2_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge2_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1724,6 +2384,48 @@ exit:
}
define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge3'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%v1>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%u0>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add3>/ir<%u0>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add2>/vp<[[VP6]]> ir<%add4>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1768,6 +2470,48 @@ exit:
}
define void @uniform_branch_unstructured_merge3_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge3_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%v1>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%u0>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1812,6 +2556,48 @@ exit:
}
define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge4'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%v1>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%v1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%u0>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<%add1>/vp<[[VP4]]> ir<%add3>/ir<%u0>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%phi2>, ir<2>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP5]]> ir<%add2>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1858,6 +2644,48 @@ exit:
}
define void @uniform_branch_unstructured_merge4_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge4_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%v1>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%v1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%u0>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1904,6 +2732,40 @@ exit:
}
define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_shared_join_with_varying'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0> ir<%add3>/vp<[[VP5]]>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1944,6 +2806,42 @@ exit:
}
define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'uniform_branch_shared_join_with_varying_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -1986,6 +2884,47 @@ exit:
}
define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) {
+; CHECK-LABEL: VPlan for loop in 'unstructured_uniform_only'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%u2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%u2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP5]]> ir<%add3>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -2030,6 +2969,47 @@ exit:
}
define void @unstructured_uniform_only_no_phi(ptr %a, i1 %u0, i1 %u2) {
+; CHECK-LABEL: VPlan for loop in 'unstructured_uniform_only_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%u2>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%u2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -2074,6 +3054,63 @@ exit:
}
define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) {
+; CHECK-LABEL: VPlan for loop in 'unstructured_uniform_only_sese_region'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%u3>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%iv>, ir<5>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%u3>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP10]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%u1>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP11]]>
+; CHECK-NEXT: Successor(s): bb7
+; CHECK-EMPTY:
+; CHECK-NEXT: bb7:
+; CHECK-NEXT: BLEND ir<%phi7> = ir<%add5>/vp<[[VP8]]> ir<%add4>/vp<[[VP11]]>
+; CHECK-NEXT: EMIT ir<%add7> = add ir<%phi7>, ir<7>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: EMIT ir<%add6> = add ir<%iv>, ir<6>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb8
+; CHECK-EMPTY:
+; CHECK-NEXT: bb8:
+; CHECK-NEXT: BLEND ir<%phi8> = ir<%add7>/vp<[[VP4]]> ir<%add6>/ir<%v0>
+; CHECK-NEXT: EMIT store ir<%phi8>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
@@ -2135,6 +3172,62 @@ exit:
}
define void @unstructured_uniform_only_sese_region_no_phi(ptr %a, i1 %u1, i1 %u3) {
+; CHECK-LABEL: VPlan for loop in 'unstructured_uniform_only_sese_region_no_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%u3>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%iv>, ir<5>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%add5>, ir<%gep>, vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%u3>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP10]]>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb7
+; CHECK-EMPTY:
+; CHECK-NEXT: bb7:
+; CHECK-NEXT: Successor(s): bb6
+; CHECK-EMPTY:
+; CHECK-NEXT: bb6:
+; CHECK-NEXT: EMIT ir<%add6> = add ir<%iv>, ir<6>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add6>, ir<%gep>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb8
+; CHECK-EMPTY:
+; CHECK-NEXT: bb8:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
entry:
br label %bb0
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index b9b41ce2b01e0..8a1afd76d60cc 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -295,3 +295,2168 @@ latch:
exit:
ret void
}
+
+;; Uniform control flow preservation tests
+define void @outermost_uniform_branch(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @outermost_uniform_branch(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP3]], <4 x i64> [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[U0]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb1 (v) \
+; / | |
+; bb2 | |
+; \ | |
+; bb3 [phi] |
+; \ /
+; bb4 [phi]
+; bb0's uniform branch can be easily preserved.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ %add0 = add i64 %iv, 0
+ br i1 %u0, label %bb1, label %bb4
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add3, %bb3 ], [ %add0, %bb0 ]
+ store i64 %phi4, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @outermost_uniform_branch_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP1]], ptr align 4 [[TMP0]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb1 (v) \
+; / | |
+; bb2 | |
+; \ | |
+; bb3 |
+; \ /
+; bb4
+; bb0's uniform branch can be easily preserved.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %add0 = add i64 %iv, 0
+ store i64 %add0, ptr %gep
+ br i1 %u0, label %bb1, label %bb4
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb3
+
+bb3:
+ br label %bb4
+
+bb4:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @outermost_uniform_branch_more_blocks(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP4]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 [phi] |
+; \ /
+; bb6 [phi]
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb6
+
+bb2:
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ %add5 = add i64 %phi5, 5
+ br label %bb6
+
+bb6:
+ %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ]
+ store i64 %phi6, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @outermost_uniform_branch_more_blocks_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 |
+; \ /
+; bb6
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb6
+
+bb2:
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ br label %bb6
+
+bb6:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
+; CHECK-LABEL: define void @uniform_branch_after_varying_branch(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb4 bb1 (u)
+; | / |
+; | bb2 |
+; | \ |
+; | bb3 [phi]
+; \ /
+; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb4, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br i1 %u1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
+; CHECK-LABEL: define void @uniform_branch_after_varying_branch_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP0]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP0]], <4 x i1> [[TMP1]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb4 bb1 (u)
+; | / |
+; | bb2 |
+; | \ |
+; | bb3
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb4, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br i1 %u1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb3
+
+bb3:
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
+; CHECK-LABEL: define void @uniform_branch_after_varying_branch_more_blocks(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb5 bb1 (u)
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4 [phi]
+; \ /
+; bb6 [phi]
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb6
+
+bb5:
+ %add5 = add i64 %iv, 5
+ br label %bb6
+
+bb6:
+ %phi6 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ]
+ store i64 %phi6, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %u1) {
+; CHECK-LABEL: define void @uniform_branch_after_varying_branch_more_blocks_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb5 bb1 (u)
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4
+; \ /
+; bb6
+; Similar to above, but with extra blocks on some edges. Probably doesn't
+; require any extra handling but nice to test explicitly.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ br label %bb6
+
+bb5:
+ %add5 = add i64 %iv, 5
+ store i64 %add5, ptr %gep
+ br label %bb6
+
+bb6:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) {
+; CHECK-LABEL: define void @uniform_branch_after_varying_branch_more_blocks_mirrored(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U2]], <4 x i64> [[TMP4]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb2 (u) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 [phi] |
+; \ /
+; bb6 [phi]
+; Mirror of the above test so that "(u)" block would be processed before/after
+; the other "(v)" destination by the RPOT.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb6
+
+bb2:
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ %add5 = add i64 %phi5, 5
+ br label %bb6
+
+bb6:
+ %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ]
+ store i64 %phi6, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr %a, i1 %u2) {
+; CHECK-LABEL: define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb2 (u) bb1
+; / \ |
+; bb4 bb3 |
+; \ / |
+; bb5 |
+; \ /
+; bb6
+; Mirror of the above test so that "(u)" block would be processed before/after
+; the other "(v)" destination by the RPOT.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb6
+
+bb2:
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ br label %bb6
+
+bb6:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) {
+; CHECK-LABEL: define void @uniform_branch_on_masked_def(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[UNI_PTR1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[UNI_PTR1]], i64 1
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[UNI_PTR1]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP2:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META16:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i1> poison, i1 [[TMP2]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
+; CHECK: [[PRED_LOAD_IF2]]:
+; CHECK-NEXT: [[TMP6:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META16]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i1> [[TMP4]], i1 [[TMP6]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE3]]
+; CHECK: [[PRED_LOAD_CONTINUE3]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ [[TMP4]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP7]], %[[PRED_LOAD_IF2]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
+; CHECK: [[PRED_LOAD_IF4]]:
+; CHECK-NEXT: [[TMP10:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META16]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i1> [[TMP8]], i1 [[TMP10]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
+; CHECK: [[PRED_LOAD_CONTINUE5]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ [[TMP8]], %[[PRED_LOAD_CONTINUE3]] ], [ [[TMP11]], %[[PRED_LOAD_IF4]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_IF6]]:
+; CHECK-NEXT: [[TMP14:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META16]]
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i1> [[TMP12]], i1 [[TMP14]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_CONTINUE7]]:
+; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i1> [ [[TMP12]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP15]], %[[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP19]], <4 x i64> [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP20]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[PREDPHI8]], i64 3
+; CHECK-NEXT: store i64 [[TMP21]], ptr [[A]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META16]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[BB0:.*]]
+; CHECK: [[BB0]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[BB5:.*]] ]
+; CHECK-NEXT: [[ADD0:%.*]] = add i64 [[IV]], 0
+; CHECK-NEXT: [[V0:%.*]] = icmp sle i64 [[IV]], 0
+; CHECK-NEXT: br i1 [[V0]], label %[[BB5]], label %[[BB1:.*]]
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[CU1:%.*]] = load i1, ptr [[UNI_PTR1]], align 1
+; CHECK-NEXT: br i1 [[CU1]], label %[[BB3:.*]], label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: [[ADD2:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: br label %[[BB4:.*]]
+; CHECK: [[BB3]]:
+; CHECK-NEXT: [[ADD3:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: br label %[[BB4]]
+; CHECK: [[BB4]]:
+; CHECK-NEXT: [[PHI4:%.*]] = phi i64 [ [[ADD2]], %[[BB2]] ], [ [[ADD3]], %[[BB3]] ]
+; CHECK-NEXT: [[ADD4:%.*]] = add i64 [[PHI4]], 4
+; CHECK-NEXT: br label %[[BB5]]
+; CHECK: [[BB5]]:
+; CHECK-NEXT: [[PHI5:%.*]] = phi i64 [ [[ADD0]], %[[BB0]] ], [ [[ADD4]], %[[BB4]] ]
+; CHECK-NEXT: store i64 [[PHI5]], ptr [[A]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[BB0]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; | bb1 (br i1 (load i1 %uni.ptr))
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4
+; \ /
+; bb5 [phi]
+; bb1's condition is uniform, but its value is loaded only along one path of
+; bb0's varying branch. It cannot be preserved trivially, because some the load
+; may never be executed.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %add0 = add i64 %iv, 0
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ %cu1 = load i1, ptr %uni.ptr1
+ br i1 %cu1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add0, %bb0 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_on_masked_def_no_phi(ptr %a, ptr %uni.ptr1) {
+; CHECK-LABEL: define void @uniform_branch_on_masked_def_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[UNI_PTR1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 1024
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[UNI_PTR1]], i64 1
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[UNI_PTR1]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP0]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META26:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META26]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x i1> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
+; CHECK: [[PRED_LOAD_IF2]]:
+; CHECK-NEXT: [[TMP7:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META26]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i1> [[TMP5]], i1 [[TMP7]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE3]]
+; CHECK: [[PRED_LOAD_CONTINUE3]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ [[TMP5]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP8]], %[[PRED_LOAD_IF2]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
+; CHECK: [[PRED_LOAD_IF4]]:
+; CHECK-NEXT: [[TMP11:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META26]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i1> [[TMP9]], i1 [[TMP11]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
+; CHECK: [[PRED_LOAD_CONTINUE5]]:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i1> [ [[TMP9]], %[[PRED_LOAD_CONTINUE3]] ], [ [[TMP12]], %[[PRED_LOAD_IF4]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_IF6]]:
+; CHECK-NEXT: [[TMP15:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META26]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i1> [[TMP13]], i1 [[TMP15]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_CONTINUE7]]:
+; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x i1> [ [[TMP13]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP16]], %[[PRED_LOAD_IF6]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = xor <4 x i1> [[TMP17]], splat (i1 true)
+; CHECK-NEXT: [[TMP19:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP18]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP20]], ptr align 4 [[TMP0]], <4 x i1> [[TMP19]]), !alias.scope [[META23]], !noalias [[META26]]
+; CHECK-NEXT: [[TMP21:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP17]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP22:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP22]], ptr align 4 [[TMP0]], <4 x i1> [[TMP21]]), !alias.scope [[META23]], !noalias [[META26]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[BB0:.*]]
+; CHECK: [[BB0]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[BB5:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[ADD0:%.*]] = add i64 [[IV]], 0
+; CHECK-NEXT: store i64 [[ADD0]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[V0:%.*]] = icmp sle i64 [[IV]], 0
+; CHECK-NEXT: br i1 [[V0]], label %[[BB5]], label %[[BB1:.*]]
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[CU1:%.*]] = load i1, ptr [[UNI_PTR1]], align 1
+; CHECK-NEXT: br i1 [[CU1]], label %[[BB3:.*]], label %[[BB2:.*]]
+; CHECK: [[BB2]]:
+; CHECK-NEXT: [[ADD2:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: store i64 [[ADD2]], ptr [[GEP]], align 4
+; CHECK-NEXT: br label %[[BB4:.*]]
+; CHECK: [[BB3]]:
+; CHECK-NEXT: [[ADD3:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: store i64 [[ADD3]], ptr [[GEP]], align 4
+; CHECK-NEXT: br label %[[BB4]]
+; CHECK: [[BB4]]:
+; CHECK-NEXT: br label %[[BB5]]
+; CHECK: [[BB5]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[BB0]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; | bb1 (br i1 (load i1 %uni.ptr))
+; | / \
+; | bb3 bb2
+; | \ /
+; | bb4
+; \ /
+; bb5
+; bb1's condition is uniform, but its value is loaded only along one path of
+; bb0's varying branch. It cannot be preserved trivially, because some the load
+; may never be executed.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %add0 = add i64 %iv, 0
+ store i64 %add0, ptr %gep
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb5, label %bb1
+
+bb1:
+ %cu1 = load i1, ptr %uni.ptr1
+ br i1 %cu1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge1(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i1> [[TMP6]], [[TMP0]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP8]], <4 x i64> [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform
+; control flow merging into the middle of the varying diamond, but "covering"
+; different potential RPOT traversals.
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; bb4 bb3 [phi]
+; \ /
+; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge1_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge1_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform
+; control flow merging into the middle of the varying diamond, but "covering"
+; different potential RPOT traversals.
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; bb4 bb3
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge2(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge2(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i1> [[TMP6]], [[TMP0]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP8]], <4 x i64> [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; | +-\---+
+; | / \
+; bb4 [phi] bb3
+; \ /
+; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb5
+
+bb4:
+ %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge2_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge2_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / \ /
+; | +-\---+
+; | / \
+; bb4 bb3
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb4, label %bb3
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb5
+
+bb4:
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge3(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i1> [[BROADCAST_SPLAT]], [[TMP6]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP8]], <4 x i64> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; bb4 [phi] bb2
+; \ /
+; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge3_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge3_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; bb4 bb2
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge4(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i1> [[BROADCAST_SPLAT]], [[TMP6]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 2)
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP8]], <4 x i64> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; +-/--+ \
+; / \ \
+; bb4 bb2 [phi]
+; \ /
+; bb5 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ %phi2 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ]
+ %add2 = add i64 %phi2, 2
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb2
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_unstructured_merge4_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_unstructured_merge4_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb3 bb1 (v)
+; \ / \
+; +-/--+ \
+; / \ \
+; bb4 bb2
+; \ /
+; bb5
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb3, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb4, label %bb2
+
+bb2:
+ br label %bb5
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb2
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_shared_join_with_varying(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP2]], <4 x i64> [[TMP1]]
+; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP5]], <4 x i64> [[PREDPHI]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / | |
+; bb3 | /
+; \ | /
+; bb4 [phi]
+; Theoretically can be done, but would require a combination of a blend and a phi.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb3, label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ], [ %add3, %bb3 ]
+ store i64 %phi4, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
+; CHECK-LABEL: define void @uniform_branch_shared_join_with_varying_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (v) bb1
+; / | /
+; bb3 | /
+; \ | /
+; bb4
+; Theoretically can be done, but would require a combination of a blend and a phi.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb4
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ %v2 = icmp sle i64 %iv, 2
+ br i1 %v2, label %bb3, label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br label %bb4
+
+bb4:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) {
+; CHECK-LABEL: define void @unstructured_uniform_only(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]], i1 [[U2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[TMP2]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP6]], <4 x i64> [[TMP5]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP8]], <4 x i64> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 3
+; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (u) bb1
+; / \ /
+; bb4 bb3 [phi]
+; \ /
+; bb5 [phi]
+; Triviallly preservable, but detection might not be easy.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3 = add i64 %phi3, 3
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ]
+ store i64 %phi5, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @unstructured_uniform_only_no_phi(ptr %a, i1 %u0, i1 %u2) {
+; CHECK-LABEL: define void @unstructured_uniform_only_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]], i1 [[U2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP2]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP2]], <4 x i1> [[BROADCAST_SPLAT2]])
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP2]], <4 x i1> [[TMP1]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (u)
+; / \
+; bb2 (u) bb1
+; / \ /
+; bb4 bb3
+; \ /
+; bb5
+; Triviallly preservable, but detection might not be easy.
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ br i1 %u0, label %bb2, label %bb1
+
+bb1:
+ %add1 = add i64 %iv, 1
+ store i64 %add1, ptr %gep
+ br label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br i1 %u2, label %bb4, label %bb3
+
+bb3:
+ br label %bb5
+
+bb4:
+ %add4 = add i64 %iv, 4
+ store i64 %add4, ptr %gep
+ br label %bb5
+
+bb5:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) {
+; CHECK-LABEL: define void @unstructured_uniform_only_sese_region(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]], i1 [[U3:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP4]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP7]], <4 x i64> [[TMP5]]
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP10]], <4 x i64> [[TMP11]], <4 x i64> [[TMP8]]
+; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i64> [[PREDPHI3]], splat (i64 7)
+; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 6)
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP12]], <4 x i64> [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[PREDPHI4]], i64 3
+; CHECK-NEXT: store i64 [[TMP14]], ptr [[A]], align 4
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb6 bb1 (u)
+; | / \
+; | bb3 (u) bb2
+; | / \ /
+; | bb5 bb4 [phi]
+; | \ /
+; | bb7 [phi]
+; \ /
+; \ /
+; bb8 [phi]
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb8 ]
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb6, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ br i1 %u3, label %bb5, label %bb4
+
+bb4:
+ %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ]
+ %add4 = add i64 %phi4, 4
+ br label %bb7
+
+bb5:
+ %add5 = add i64 %iv, 5
+ br label %bb7
+
+bb6:
+ %add6 = add i64 %iv, 6
+ br label %bb8
+
+bb7:
+ %phi7 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ]
+ %add7 = add i64 %phi7, 7
+ br label %bb8
+
+bb8:
+ %phi8 = phi i64 [ %add6, %bb6 ], [ %add7, %bb7 ]
+ store i64 %phi8, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @unstructured_uniform_only_sese_region_no_phi(ptr %a, i1 %u1, i1 %u3) {
+; CHECK-LABEL: define void @unstructured_uniform_only_sese_region_no_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]], i1 [[U3:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[U3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP9]], ptr align 4 [[TMP1]], <4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 6)
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP10]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %bb0
+
+bb0:
+; bb0 (v)
+; / \
+; bb6 bb1 (u)
+; | / \
+; | bb3 (u) bb2
+; | / \ /
+; | bb5 bb4
+; | \ /
+; | bb7
+; \ /
+; \ /
+; bb8
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb8 ]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %v0 = icmp sle i64 %iv, 0
+ br i1 %v0, label %bb6, label %bb1
+
+bb1:
+ br i1 %u1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i64 %iv, 2
+ store i64 %add2, ptr %gep
+ br label %bb4
+
+bb3:
+ %add3 = add i64 %iv, 3
+ store i64 %add3, ptr %gep
+ br i1 %u3, label %bb5, label %bb4
+
+bb4:
+ br label %bb7
+
+bb5:
+ %add5 = add i64 %iv, 5
+ store i64 %add5, ptr %gep
+ br label %bb7
+
+bb6:
+ %add6 = add i64 %iv, 6
+ store i64 %add6, ptr %gep
+ br label %bb8
+
+bb7:
+ br label %bb8
+
+bb8:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
More information about the llvm-branch-commits
mailing list