[llvm-branch-commits] [llvm] [VPlan] Peek through nested phi incoming values in computeBlendMasks (PR #237152)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 8 21:48:32 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/237152
>From 17877eebe624c4be06e53385b378f4a6e25b1100 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 26 Oct 2026 17:51:51 +0800
Subject: [PATCH] [VPlan] Peek through nested phi incoming values in
computeBlendMasks
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 91 +++-
.../LoopVectorize/VPlan/predicator.ll | 395 +++++++++++++-
.../Transforms/LoopVectorize/find-last.ll | 81 +++
.../Transforms/LoopVectorize/predicator.ll | 495 ++++++++++++++++++
.../LoopVectorize/select-cmp-blend-chain.ll | 16 +-
.../Transforms/LoopVectorize/uniform-blend.ll | 15 +-
6 files changed, 1061 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 1301c566114ded..58c00e3b4196e6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -83,15 +83,37 @@ class VPPredicator {
return VPBB->getFirstNonPhi();
}
+ /// Returns where to insert new blends in \p VPBB.
+ VPBasicBlock::iterator getBlendInsertPoint(VPBasicBlock *VPBB) {
+ VPBasicBlock::iterator It = getMaskInsertPoint(VPBB);
+ if (It == VPBB->end())
+ return It;
+ // Insert after any incoming edge masks if they were already inserted.
+ for (VPBlockBase *Pred : VPBB->predecessors())
+ if (VPValue *EdgeMask = getEdgeMask(cast<VPBasicBlock>(Pred), VPBB))
+ if (VPRecipeBase *EdgeR = EdgeMask->getDefiningRecipe())
+ if (EdgeR->getParent() == VPBB &&
+ !VPDT.properlyDominates(EdgeR, &*It)) {
+ It = std::next(EdgeR->getIterator());
+ if (It == VPBB->end())
+ return It;
+ }
+ return It;
+ }
+
using EdgeTy = std::pair<const VPBasicBlock *, const VPBasicBlock *>;
/// Compute the set of edges that are "furthest up" in the CFG for each
/// incoming value of \p Phi.
MapVector<EdgeTy, VPValue *> computeBlendEdges(VPPhi *Phi);
- /// Given a set of \p Edges that each can reach \p VPBB, return the OR of all
- /// edges, or an equivalent block in-mask.
- VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+ /// Given a set of \p Edges that each can reach \p VPBB, try to see if there
+ /// is a block in-mask that can be used for its blend mask. Only valid when
+ /// there's exactly one unique incoming value on the paths from Edges to VPBB.
+ VPValue *tryFindBlockInMask(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+
+ /// Return the OR of the edge masks of \p Edges.
+ VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges);
public:
VPPredicator(VPlan &Plan)
@@ -281,6 +303,15 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
AddEdge(InVPBB, Phi->getParent(), InVal);
+ // Don't optimize any phis used to mask tail values of reductions during tail
+ // folding until handleFindLastReductions can match the new form.
+ VPlan *Plan = Phi->getParent()->getPlan();
+ if (Phi->getParent() == Plan->getVectorLoopRegion()->getExiting() &&
+ any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>))
+ return Edges;
+
+ // Phis whose incoming values have been added to Edges.
+ SmallPtrSet<const VPUser *, 4> Expanded = {Phi};
SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks());
while (!Worklist.empty()) {
auto *VPBB = cast<VPBasicBlock>(Worklist.pop_back_val());
@@ -299,6 +330,24 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (EdgeTy Edge : OutEdges)
Edges.erase(Edge);
+ // If the value is a phi postdominated by VPBB which shares an incoming
+ // value in Edges, then look through the inner incoming values instead of
+ // propagating the phi. This deduplicates a select.
+ if (auto *InnerPhi = dyn_cast<VPPhi>(Common))
+ if (all_of(InnerPhi->users(),
+ [&Expanded](VPUser *U) { return Expanded.contains(U); }) &&
+ VPPDT.dominates(VPBB, InnerPhi->getParent()) &&
+ any_of(InnerPhi->incoming_values(), [&Edges](VPValue *InV) {
+ return is_contained(make_second_range(Edges), InV);
+ })) {
+ Expanded.insert(InnerPhi);
+ for (auto [InV, InVPBB] : InnerPhi->incoming_values_and_blocks()) {
+ AddEdge(InVPBB, InnerPhi->getParent(), InV);
+ Worklist.insert(InVPBB);
+ }
+ continue;
+ }
+
// Iterate up through the post dominance frontier.
assert(VPPDF.find(VPBB) != VPPDF.end() &&
"VPBB must have a post-dominance frontier entry");
@@ -315,8 +364,8 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
return Edges;
}
-VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
- VPBasicBlock *VPBB) {
+VPValue *VPPredicator::tryFindBlockInMask(ArrayRef<EdgeTy> Edges,
+ VPBasicBlock *VPBB) {
// If the nearest common postdominator to all of Edges destinations isn't VPBB
// then we can use its block in-mask. E.g:
//
@@ -329,16 +378,19 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
// VPBB
//
// If the edges are A->D and B->C, PostDom will be D. We can reuse Ds block
- // in-mask.
+ // in-mask. Only valid when there is only one unique incoming value flowing
+ // from the edges to VPBB.
const VPBasicBlock *PostDom = Edges[0].second;
for (auto [_, DstVPBB] : drop_begin(Edges))
PostDom =
cast<VPBasicBlock>(VPPDT.findNearestCommonDominator(PostDom, DstVPBB));
assert(VPPDT.dominates(VPBB, PostDom) && "VPBB doesn't postdominate edges");
- if (PostDom != VPBB)
- return getBlockInMask(PostDom);
+ if (PostDom == VPBB)
+ return nullptr;
+ return getBlockInMask(PostDom);
+}
- // Otherwise, compute the disjunction of edges.
+VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges) {
VPValue *Mask = nullptr;
for (auto [Src, ConstDst] : Edges) {
auto *Dst = const_cast<VPBasicBlock *>(ConstDst);
@@ -354,7 +406,7 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
}
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
- Builder.setInsertPoint(getMaskInsertPoint(VPBB));
+ Builder.setInsertPoint(getBlendInsertPoint(VPBB));
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
@@ -379,8 +431,19 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
MapVector<VPValue *, SmallVector<EdgeTy>> InValEdgesMap;
for (auto [Edge, Val] : computeBlendEdges(PhiR))
InValEdgesMap[Val].push_back(Edge);
+ bool IncomingEdgesUnique =
+ all_of(PhiR->incoming_values(), [&](VPValue *InV) {
+ return is_contained(InValEdgesMap.keys(), InV);
+ });
+
auto InValEdges = InValEdgesMap.takeVector();
+ if (InValEdges.size() == 1) {
+ PhiR->replaceAllUsesWith(InValEdges[0].first);
+ PhiR->eraseFromParent();
+ continue;
+ }
+
// Sort the incoming value order to match PhiR as much as possible.
llvm::stable_sort(InValEdges, [&PhiR](auto &L, auto &R) {
auto InVs = PhiR->incoming_values();
@@ -391,7 +454,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
SmallVector<VPValue *, 2> OperandsWithMask;
for (const auto &[InVPV, Edges] : InValEdges) {
OperandsWithMask.push_back(InVPV);
- OperandsWithMask.push_back(createBlendMaskForEdges(Edges, VPBB));
+ if (IncomingEdgesUnique) {
+ if (auto *Mask = tryFindBlockInMask(Edges, VPBB)) {
+ OperandsWithMask.push_back(Mask);
+ continue;
+ }
+ }
+ OperandsWithMask.push_back(createBlendMaskForEdges(Edges));
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
auto *Blend =
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 46887ec305304c..e8629499763db8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -665,8 +665,6 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): merge.a
; CHECK-EMPTY:
; CHECK-NEXT: merge.a:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: BLEND ir<%blend.a> = ir<%v1>/ir<%c0> ir<%v1>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): if.b
; CHECK-EMPTY:
@@ -675,16 +673,14 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): if.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: if.b.inner:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
; CHECK-NEXT: Successor(s): merge.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: merge.b.inner:
; CHECK-NEXT: Successor(s): merge.b
; CHECK-EMPTY:
; CHECK-NEXT: merge.b:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%d0>
-; CHECK-NEXT: BLEND ir<%blend.b> = ir<%v2>/ir<%d0> ir<%v2>/vp<[[VP7]]>
-; CHECK-NEXT: EMIT ir<%sum> = add ir<%blend.a>, ir<%blend.b>
+; CHECK-NEXT: EMIT ir<%sum> = add ir<%v1>, ir<%v2>
; CHECK-NEXT: EMIT store ir<%sum>, ir<%gep>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
@@ -1042,3 +1038,390 @@ latch:
exit:
ret void
}
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%x>/ir<%c0> ir<%y>/vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_multi_use'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ store i32 %phi4, ptr %gep
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Don't look through %phi3 as it doesn't share any incoming values with %phi4.
+define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, i32 %z, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_no_shared_values'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%z>/vp<[[VP5]]> ir<%y>/vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%phi3>/vp<[[VP4]]> ir<%x>/ir<%c0>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; | \
+; | bb1
+; | | \
+; | | bb2
+; | | /
+; | bb3
+; | /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb4]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb2, label %bb3
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [%y, %bb1], [%z, %bb2]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi4, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_blend_insert_pos'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<2>/vp<[[VP6]]> ir<1>/vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%c0>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP8]]> ir<2>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / \ \
+; bb2 | /
+; \ / /
+; bb3 /
+; \ /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret i32 %phi4
+}
+
+define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) {
+; CHECK-LABEL: VPlan for loop in 'diamond_phi2_look_thru_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP5]]> ir<0>/vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]>
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<3>/vp<[[VP11]]> ir<2>/vp<[[VP9]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 bb2
+; | \ / |
+; | bb4 |
+; \ | /
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [0, %entry], [%iv.next, %bb5]
+ %c0 = icmp sle i64 %iv, 0
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br i1 %c1, label %bb4, label %bb5
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br i1 %c2, label %bb4, label %bb5
+
+bb4:
+ %phi4 = phi i64 [0, %bb1], [1, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ store i64 %phi5, ptr %gep
+ %iv.next = add nsw nuw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/find-last.ll b/llvm/test/Transforms/LoopVectorize/find-last.ll
index 9948f34e4571b6..35cf71099b101c 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last.ll
@@ -445,3 +445,84 @@ loop.latch:
exit:
ret i32 %rdx.next
}
+
+define i32 @find_last_phi_of_two_updates(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define i32 @find_last_phi_of_two_updates(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 5)
+; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 2)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[WIDE_LOAD1]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
+; CHECK-NEXT: [[TMP13]] = select i1 [[TMP12]], <4 x i1> [[TMP11]], <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP14]] = select i1 [[TMP12]], <4 x i32> [[PREDPHI]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP14]], <4 x i1> [[TMP13]], i32 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %rdx = phi i32 [ -1, %entry ], [ %rdx.next, %loop.latch ]
+ %gep.a = getelementptr i32, ptr %a, i64 %iv
+ %ld.a = load i32, ptr %gep.a
+ %gep.b = getelementptr i32, ptr %b, i64 %iv
+ %ld.b = load i32, ptr %gep.b
+ %c.0 = icmp sgt i32 %ld.a, 5
+ br i1 %c.0, label %then, label %else
+
+then:
+ %c.1 = icmp ne i32 %ld.b, 0
+ br i1 %c.1, label %merge, label %update
+
+else:
+ %c.2 = icmp ne i32 %ld.a, 2
+ br i1 %c.2, label %merge, label %update
+
+update:
+ %val = phi i32 [ %ld.b, %then ], [ %ld.a, %else ]
+ br label %merge
+
+merge:
+ %rdx.next = phi i32 [ %rdx, %then ], [ %rdx, %else ], [ %val, %update ]
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret i32 %rdx.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 8a7b4fb2f71816..53d853dd86dff9 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -362,3 +362,498 @@ latch:
exit:
ret void
}
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_multi_use(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT7]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[PREDPHI]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ store i32 %phi4, ptr %gep
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Don't look through %phi3 as it doesn't share any incoming values with %phi4.
+define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, i32 %z, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_no_shared_values(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Z]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI5]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; | \
+; | bb1
+; | | \
+; | | bb2
+; | | /
+; | bb3
+; | /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb4]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb2, label %bb3
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [%y, %bb1], [%z, %bb2]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi4, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @look_thru_phi_blend_insert_pos(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> splat (i32 2), <4 x i32> splat (i32 1)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+; bb0
+; / \
+; bb1 \
+; / \ \
+; bb2 | /
+; \ / /
+; bb3 /
+; \ /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret i32 %phi4
+}
+
+define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @diamond_phi2_look_thru_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> zeroinitializer, <4 x i64> splat (i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> splat (i64 3), <4 x i64> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> splat (i64 2), <4 x i64> [[PREDPHI3]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[PREDPHI4]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 bb2
+; | \ / |
+; | bb4 |
+; \ | /
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [0, %entry], [%iv.next, %bb5]
+ %c0 = icmp sle i64 %iv, 0
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br i1 %c1, label %bb4, label %bb5
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br i1 %c2, label %bb4, label %bb5
+
+bb4:
+ %phi4 = phi i64 [0, %bb1], [1, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ store i64 %phi5, ptr %gep
+ %iv.next = add nsw nuw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @blend_look_through_no_shared_values(ptr noalias %dst, ptr noalias %conds, i32 %x, i32 %y, i32 %u, i32 %w, i64 %n) {
+; CHECK-LABEL: define void @blend_look_through_no_shared_values(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[CONDS:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[U:%.*]], i32 [[W:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[U]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[W]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i8> [[WIDE_LOAD]] to <4 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 2)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i8> [[TMP5]] to <4 x i1>
+; CHECK-NEXT: [[TMP7:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 3)
+; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i8> [[TMP7]] to <4 x i1>
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP13]], <4 x i32> [[BROADCAST_SPLAT4]], <4 x i32> [[PREDPHI7]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI8]], ptr [[TMP14]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_CONDS:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i8, ptr [[GEP_CONDS]], align 1
+; CHECK-NEXT: [[C_0:%.*]] = trunc i8 [[LD]] to i1
+; CHECK-NEXT: [[SHR_1:%.*]] = lshr i8 [[LD]], 1
+; CHECK-NEXT: [[C_1:%.*]] = trunc i8 [[SHR_1]] to i1
+; CHECK-NEXT: [[SHR_2:%.*]] = lshr i8 [[LD]], 2
+; CHECK-NEXT: [[C_2:%.*]] = trunc i8 [[SHR_2]] to i1
+; CHECK-NEXT: [[SHR_3:%.*]] = lshr i8 [[LD]], 3
+; CHECK-NEXT: [[C_3:%.*]] = trunc i8 [[SHR_3]] to i1
+; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br i1 [[C_1]], label %[[THEN_THEN:.*]], label %[[THEN_ELSE:.*]]
+; CHECK: [[THEN_THEN]]:
+; CHECK-NEXT: br i1 [[C_2]], label %[[USE_X:.*]], label %[[USE_Y:.*]]
+; CHECK: [[THEN_ELSE]]:
+; CHECK-NEXT: br label %[[THEN_ELSE_CONT:.*]]
+; CHECK: [[THEN_ELSE_CONT]]:
+; CHECK-NEXT: br label %[[USE_X]]
+; CHECK: [[USE_X]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[USE_Y]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: br i1 [[C_3]], label %[[ELSE_THEN:.*]], label %[[ELSE_ELSE:.*]]
+; CHECK: [[ELSE_THEN]]:
+; CHECK-NEXT: br label %[[ELSE_MERGE:.*]]
+; CHECK: [[ELSE_ELSE]]:
+; CHECK-NEXT: br label %[[ELSE_MERGE]]
+; CHECK: [[ELSE_MERGE]]:
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[U]], %[[ELSE_THEN]] ], [ [[W]], %[[ELSE_ELSE]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[X]], %[[USE_X]] ], [ [[Y]], %[[USE_Y]] ], [ [[P]], %[[ELSE_MERGE]] ]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.conds = getelementptr i8, ptr %conds, i64 %iv
+ %ld = load i8, ptr %gep.conds
+ %c.0 = trunc i8 %ld to i1
+ %shr.1 = lshr i8 %ld, 1
+ %c.1 = trunc i8 %shr.1 to i1
+ %shr.2 = lshr i8 %ld, 2
+ %c.2 = trunc i8 %shr.2 to i1
+ %shr.3 = lshr i8 %ld, 3
+ %c.3 = trunc i8 %shr.3 to i1
+ br i1 %c.0, label %then, label %else
+
+then:
+ br i1 %c.1, label %then.then, label %then.else
+
+then.then:
+ br i1 %c.2, label %use.x, label %use.y
+
+then.else:
+ br label %then.else.cont
+
+then.else.cont:
+ br label %use.x
+
+use.x:
+ br label %loop.latch
+
+use.y:
+ br label %loop.latch
+
+else:
+ br i1 %c.3, label %else.then, label %else.else
+
+else.then:
+ br label %else.merge
+
+else.else:
+ br label %else.merge
+
+else.merge:
+ %p = phi i32 [ %u, %else.then ], [ %w, %else.else ]
+ br label %loop.latch
+
+loop.latch:
+ %merge = phi i32 [ %x, %use.x ], [ %y, %use.y ], [ %p, %else.merge ]
+ %gep.dst = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %merge, ptr %gep.dst
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
index 6cae52e2a8f283..e1fa8237cb9633 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
@@ -97,15 +97,25 @@ define i32 @anyof_three_blend_chain(i1 %c0, i1 %c1, i1 %c2, i32 %n) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[SMAX]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C2]], <4 x i1> [[VEC_PHI]], <4 x i1> [[TMP3]]
-; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[C1]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI1:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP8]]
; CHECK-NEXT: [[PREDPHI2]] = select i1 [[C0]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 8d86c8ca2bd9d7..a016a26d479ec7 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -96,19 +96,10 @@ define void @blend_chain_iv(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP7]]
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP4]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP6]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP8]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP8]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
More information about the llvm-branch-commits
mailing list