[llvm] [VPlan] Compute blend masks from minimum set of edge masks (PR #201783)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 5 01:38:31 PDT 2026
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/201783
Another PR aims to model the control flow of early exits explicitly, and in doing so insert phi nodes to preserve SSA. Inserting phi nodes results in more VPBlendRecipes, so this PR optimizes the edge masks generated for those blend recipes to prevent regressions.
The shape of the CFG and the phis that would be emitted are precomitted in the predicator-early-exit.ll test.
The idea is to compute a minimal set of edges that lead to each unique incoming value in a phi. It does this by iterating up the edges in the post dominance frontier till the outgoing edges no longer lead to the same value. It also recursively looks through the incoming edges of any values that are phi nodes themselves.
This is a simpler, less general version of #184838 since this requires the phi node to postdominate its incoming values. This is fine the early exit use case though, since we only need to optimize phi nodes inserted in the latch.
The big advantage over #184838 is that it doesn't require several depth-first searches to compute the set of reachable nodes, and can be done entirely by iterating the post-dominator frontier.
>From 5d1589e7c044e7f9460bc44b719e10c10654f3ae Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 2 Jun 2026 18:15:25 +0800
Subject: [PATCH 1/3] Insert blends in post order
---
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 2717b80e2eeaa..b01fef556f8ed 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -69,6 +69,8 @@ class VPPredicator {
return EdgeMaskCache[{Src, Dst}] = Mask;
}
+ DenseMap<const VPBasicBlock *, VPBasicBlock::iterator> InsertPoints;
+
public:
VPPredicator(VPlan &Plan) : VPDT(Plan), VPPDT(Plan) {}
@@ -136,6 +138,10 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// Start inserting after the block's phis, which be replaced by blends later.
Builder.setInsertPoint(VPBB, VPBB->getFirstNonPhi());
+ // Keep track of where in VPBB we are inserting the masks into.
+ scope_exit UpdateInsertPoint(
+ [this, &VPBB]() { InsertPoints[VPBB] = Builder.getInsertPoint(); });
+
// Reuse the mask of the immediate dominator if the VPBB post-dominates the
// immediate dominator.
auto *IDom = VPDT.getNode(VPBB)->getIDom();
@@ -225,6 +231,7 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
}
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
+ Builder.setInsertPoint(VPBB, InsertPoints[VPBB]);
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
Phis.push_back(cast<VPPhi>(&R));
@@ -276,10 +283,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
// Introduce the mask for VPBB, which may introduce needed edge masks, and
// convert all phi recipes of VPBB to blend recipes unless VPBB is the
// header.
- if (VPBB != Header) {
+ if (VPBB != Header)
Predicator.createBlockInMask(VPBB);
- Predicator.convertPhisToBlends(VPBB);
- }
VPValue *BlockMask = Predicator.getBlockInMask(VPBB);
if (!BlockMask)
@@ -292,6 +297,10 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
}
}
+ for (VPBlockBase *VPB : reverse(RPOT))
+ if (VPB != Header)
+ Predicator.convertPhisToBlends(cast<VPBasicBlock>(VPB));
+
// Linearize the blocks of the loop into one serial chain.
VPBlockBase *PrevVPBB = nullptr;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
>From cb5ad8314919e2fc380cdbe85bb6750e932d312c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 2 Jun 2026 18:01:09 +0800
Subject: [PATCH 2/3] Precommit test case
---
.../VPlan/predicator-early-exit.ll | 337 ++++++++++++++++++
1 file changed, 337 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll
new file mode 100644
index 0000000000000..674ab702be327
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -disable-output < %s -p loop-vectorize -vplan-print-after=introduceMasksAndLinearize -vplan-print-vector-region-scope 2>&1 | FileCheck %s
+
+; Test various CFGs generated by early exits
+
+; vector.body
+; / \
+; / \
+; exiting1 \
+; / \ \
+; / \ |
+; / join1 |
+; / / \ |
+; / exiting2 \ |
+; \ | \ \ |
+; \ | join2
+; \ | /
+; \ | /
+; latch
+define void @multi_early_exit_predicated_nested(ptr %p1, ptr %p2, i1 %c1, i1 %c2, i1 %ee1, i1 %ee2, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'multi_early_exit_predicated_nested'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): exiting1
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting1:
+; CHECK-NEXT: Successor(s): join1
+; CHECK-EMPTY:
+; CHECK-NEXT: join1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%ee1>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): exiting2
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting2:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%c2>
+; CHECK-NEXT: Successor(s): join2
+; CHECK-EMPTY:
+; CHECK-NEXT: join2:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%ee2>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP11]]>, vp<[[VP12]]>
+; CHECK-NEXT: BLEND ir<%phi1.join2> = ir<1>/vp<[[VP8]]> ir<1>/vp<[[VP10]]> ir<0>/vp<[[VP12]]>
+; CHECK-NEXT: BLEND ir<%phi2.join2> = ir<1>/vp<[[VP8]]> ir<0>/vp<[[VP10]]> ir<0>/vp<[[VP12]]>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%ee2>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and ir<%c1>, ir<%ee1>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<%phi1.join2>/vp<[[VP13]]> ir<1>/vp<[[VP14]]> ir<1>/vp<[[VP15]]>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<%phi2.join2>/vp<[[VP13]]> ir<1>/vp<[[VP14]]> ir<poison>/vp<[[VP15]]>
+; CHECK-NEXT: EMIT ir<%gep1> = getelementptr ir<%p1>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi1>, ir<%gep1>
+; CHECK-NEXT: EMIT ir<%gep2> = getelementptr ir<%p2>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi2>, ir<%gep2>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %latch]
+ br i1 %c1, label %exiting1, label %join2
+
+exiting1:
+ br i1 %ee1, label %latch, label %join1
+
+join1:
+ br i1 %c2, label %exiting2, label %join2
+
+exiting2:
+ br i1 %ee2, label %latch, label %join2
+
+join2:
+ %phi1.join2 = phi i32 [1, %exiting2], [1, %join1], [0, %loop]
+ %phi2.join2 = phi i32 [1, %exiting2], [0, %join1], [0, %loop]
+ br label %latch
+
+latch:
+ %phi1 = phi i32 [1, %exiting1], [1, %exiting2], [%phi1.join2, %join2]
+ %phi2 = phi i32 [poison, %exiting1], [1, %exiting2], [%phi2.join2, %join2]
+ %gep1 = getelementptr i32, ptr %p1, i32 %iv
+ store i32 %phi1, ptr %gep1
+ %gep2 = getelementptr i32, ptr %p2, i32 %iv
+ store i32 %phi2, ptr %gep2
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; vector.body
+; / \
+; / \
+; exiting1 /
+; / \ /
+; / \ /
+; / join1
+; / / \
+; / exiting2 \
+; \ | \ \
+; \ | join2
+; \ | /
+; \ | /
+; latch
+define void @multi_early_exit_predicated_not_nested(ptr %p1, ptr %p2, i1 %c1, i1 %c2, i1 %ee1, i1 %ee2, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'multi_early_exit_predicated_not_nested'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): exiting1
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting1:
+; CHECK-NEXT: Successor(s): join1
+; CHECK-EMPTY:
+; CHECK-NEXT: join1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%ee1>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi.join1> = ir<1>/vp<[[VP5]]> ir<0>/vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): exiting2
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting2:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c2>
+; CHECK-NEXT: Successor(s): join2
+; CHECK-EMPTY:
+; CHECK-NEXT: join2:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%ee2>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP8]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP10]]>, vp<[[VP12]]>
+; CHECK-NEXT: BLEND ir<%phi.join2> = ir<1>/vp<[[VP10]]> ir<0>/vp<[[VP12]]>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%ee2>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and ir<%c1>, ir<%ee1>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<%phi.join1>/vp<[[VP13]]> ir<%phi.join1>/vp<[[VP14]]> ir<1>/vp<[[VP15]]>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<%phi.join2>/vp<[[VP13]]> ir<1>/vp<[[VP14]]> ir<poison>/vp<[[VP15]]>
+; CHECK-NEXT: EMIT ir<%gep1> = getelementptr ir<%p1>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi1>, ir<%gep1>
+; CHECK-NEXT: EMIT ir<%gep2> = getelementptr ir<%p2>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi2>, ir<%gep2>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %latch]
+ br i1 %c1, label %exiting1, label %join1
+
+exiting1:
+ br i1 %ee1, label %latch, label %join1
+
+join1:
+ %phi.join1 = phi i32 [1, %exiting1], [0, %loop]
+ br i1 %c2, label %exiting2, label %join2
+
+exiting2:
+ br i1 %ee2, label %latch, label %join2
+
+join2:
+ %phi.join2 = phi i32 [1, %exiting2], [0, %join1]
+ br label %latch
+
+latch:
+ %phi1 = phi i32 [1, %exiting1], [%phi.join1, %exiting2], [%phi.join1, %join2]
+ %phi2 = phi i32 [poison, %exiting1], [1, %exiting2], [%phi.join2, %join2]
+ %gep1 = getelementptr i32, ptr %p1, i32 %iv
+ store i32 %phi1, ptr %gep1
+ %gep2 = getelementptr i32, ptr %p2, i32 %iv
+ store i32 %phi2, ptr %gep2
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; vector.body
+; / \
+; / \
+; exiting1 exiting2
+; / \ / \
+; / \ / \
+; / join1 \
+; / / \ \
+; / exiting3 exiting4 /
+; \ \ \ / / /
+; \ \ join2 / /
+; \ \ | / /
+; +---latch-----+
+define void @four_exits_2x2_diamond(ptr %p1, ptr %p2, ptr %p3, ptr %p4, i1 %c1, i1 %c2, i1 %ee1, i1 %ee2, i1 %ee3, i1 %ee4, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'four_exits_2x2_diamond'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): exiting2
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: Successor(s): exiting1
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting1:
+; CHECK-NEXT: Successor(s): join1
+; CHECK-EMPTY:
+; CHECK-NEXT: join1:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%ee2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%ee1>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = or vp<[[VP6]]>, vp<[[VP8]]>
+; CHECK-NEXT: BLEND ir<%phi1.join1> = ir<0>/vp<[[VP6]]> ir<1>/vp<[[VP8]]>
+; CHECK-NEXT: BLEND ir<%phi2.join1> = ir<1>/vp<[[VP6]]> ir<0>/vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): exiting4
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting4:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: Successor(s): exiting3
+; CHECK-EMPTY:
+; CHECK-NEXT: exiting3:
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP9]]>, ir<%c2>
+; CHECK-NEXT: Successor(s): join2
+; CHECK-EMPTY:
+; CHECK-NEXT: join2:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%ee4>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = not ir<%ee3>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = logical-and vp<[[VP12]]>, vp<[[VP15]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = or vp<[[VP14]]>, vp<[[VP16]]>
+; CHECK-NEXT: BLEND ir<%phi3.join2> = ir<0>/vp<[[VP14]]> ir<1>/vp<[[VP16]]>
+; CHECK-NEXT: BLEND ir<%phi4.join2> = ir<1>/vp<[[VP14]]> ir<0>/vp<[[VP16]]>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%ee4>
+; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP12]]>, ir<%ee3>
+; CHECK-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%ee2>
+; CHECK-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and ir<%c1>, ir<%ee1>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<%phi1.join1>/vp<[[VP17]]> ir<%phi1.join1>/vp<[[VP18]]> ir<%phi1.join1>/vp<[[VP19]]> ir<0>/vp<[[VP20]]> ir<1>/vp<[[VP21]]>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<%phi2.join1>/vp<[[VP17]]> ir<%phi2.join1>/vp<[[VP18]]> ir<%phi2.join1>/vp<[[VP19]]> ir<1>/vp<[[VP20]]> ir<poison>/vp<[[VP21]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%phi3.join2>/vp<[[VP17]]> ir<0>/vp<[[VP18]]> ir<1>/vp<[[VP19]]> ir<poison>/vp<[[VP20]]> ir<poison>/vp<[[VP21]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%phi4.join2>/vp<[[VP17]]> ir<1>/vp<[[VP18]]> ir<poison>/vp<[[VP19]]> ir<poison>/vp<[[VP20]]> ir<poison>/vp<[[VP21]]>
+; CHECK-NEXT: EMIT ir<%gep1> = getelementptr ir<%p1>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi1>, ir<%gep1>
+; CHECK-NEXT: EMIT ir<%gep2> = getelementptr ir<%p2>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi2>, ir<%gep2>
+; CHECK-NEXT: EMIT ir<%gep3> = getelementptr ir<%p3>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi3>, ir<%gep3>
+; CHECK-NEXT: EMIT ir<%gep4> = getelementptr ir<%p4>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep4>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %latch]
+ br i1 %c1, label %exiting1, label %exiting2
+
+exiting1:
+ br i1 %ee1, label %latch, label %join1
+
+exiting2:
+ br i1 %ee2, label %latch, label %join1
+
+join1:
+ %phi1.join1 = phi i32 [1, %exiting1], [0, %exiting2]
+ %phi2.join1 = phi i32 [0, %exiting1], [1, %exiting2]
+ br i1 %c2, label %exiting3, label %exiting4
+
+exiting3:
+ br i1 %ee3, label %latch, label %join2
+
+exiting4:
+ br i1 %ee4, label %latch, label %join2
+
+join2:
+ %phi3.join2 = phi i32 [1, %exiting3], [0, %exiting4]
+ %phi4.join2 = phi i32 [0, %exiting3], [1, %exiting4]
+ br label %latch
+
+latch:
+ %phi1 = phi i32 [1, %exiting1], [0, %exiting2], [%phi1.join1, %exiting3], [%phi1.join1, %exiting4], [%phi1.join1, %join2]
+ %phi2 = phi i32 [poison, %exiting1], [1, %exiting2], [%phi2.join1, %exiting3], [%phi2.join1, %exiting4], [%phi2.join1, %join2]
+ %phi3 = phi i32 [poison, %exiting1], [poison, %exiting2], [1, %exiting3], [0, %exiting4], [%phi3.join2, %join2]
+ %phi4 = phi i32 [poison, %exiting1], [poison, %exiting2], [poison, %exiting3], [1, %exiting4], [%phi4.join2, %join2]
+ %gep1 = getelementptr i32, ptr %p1, i32 %iv
+ store i32 %phi1, ptr %gep1
+ %gep2 = getelementptr i32, ptr %p2, i32 %iv
+ store i32 %phi2, ptr %gep2
+ %gep3 = getelementptr i32, ptr %p3, i32 %iv
+ store i32 %phi3, ptr %gep3
+ %gep4 = getelementptr i32, ptr %p4, i32 %iv
+ store i32 %phi4, ptr %gep4
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 375dd0d28589f71f1fbaa27d702ca8a3d6249bf5 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 3 Jun 2026 23:55:02 +0800
Subject: [PATCH 3/3] Simplify blend masks
---
.../include/llvm/Analysis/DominanceFrontier.h | 1 +
.../Transforms/Vectorize/VPlanDominatorTree.h | 8 +
.../Transforms/Vectorize/VPlanPredicator.cpp | 147 +++++++++++++++++-
.../VPlan/predicator-early-exit.ll | 30 ++--
.../LoopVectorize/VPlan/predicator.ll | 2 +-
.../LoopVectorize/predicate-switch.ll | 9 +-
.../LoopVectorize/reduction-inloop-pred.ll | 10 +-
.../LoopVectorize/reduction-inloop.ll | 21 +--
.../Transforms/LoopVectorize/reduction.ll | 10 +-
9 files changed, 189 insertions(+), 49 deletions(-)
diff --git a/llvm/include/llvm/Analysis/DominanceFrontier.h b/llvm/include/llvm/Analysis/DominanceFrontier.h
index fd38891e901e3..4a8ab96cf71a7 100644
--- a/llvm/include/llvm/Analysis/DominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/DominanceFrontier.h
@@ -78,6 +78,7 @@ class DominanceFrontierBase {
const_iterator end() const { return Frontiers.end(); }
iterator find(BlockT *B) { return Frontiers.find(B); }
const_iterator find(BlockT *B) const { return Frontiers.find(B); }
+ const_iterator find(const BlockT *B) const { return Frontiers.find(B); }
/// print - Convert to human readable form
///
diff --git a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
index 2864670f44913..1ad522880c709 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
@@ -18,6 +18,8 @@
#include "VPlan.h"
#include "VPlanCFG.h"
#include "llvm/ADT/GraphTraits.h"
+#include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/Analysis/DominanceFrontierImpl.h"
#include "llvm/IR/Dominators.h"
#include "llvm/Support/GenericDomTree.h"
#include "llvm/Support/GenericDomTreeConstruction.h"
@@ -67,5 +69,11 @@ template <>
struct GraphTraits<const VPDomTreeNode *>
: public DomTreeGraphTraitsBase<const VPDomTreeNode,
VPDomTreeNode::const_iterator> {};
+
+class VPPostDominanceFrontier
+ : public DominanceFrontierBase<VPBlockBase, true> {
+public:
+ explicit VPPostDominanceFrontier(const DomTreeT &VPDT) { analyze(VPDT); }
+};
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANDOMINATORTREE_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index b01fef556f8ed..2ec3df8ccf8c1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -34,6 +34,9 @@ class VPPredicator {
/// Post-dominator tree for the VPlan.
VPPostDominatorTree VPPDT;
+ /// Post-dominator frontier for the VPlan.
+ VPPostDominanceFrontier VPPDF;
+
/// When we if-convert we need to create edge masks. We have to cache values
/// so that we don't end up with exponential recursion/IR.
using EdgeMaskCacheTy =
@@ -69,10 +72,19 @@ class VPPredicator {
return EdgeMaskCache[{Src, Dst}] = Mask;
}
+ using EdgeTy = std::pair<const VPBasicBlock *, const VPBasicBlock *>;
+
+ /// Compute the "furthest up" set of edges for each incoming value of \Phi.
+ MapVector<EdgeTy, VPValue *> computeBlendEdges(VPPhi *Phi);
+
+ /// Given a set of \p Edges that lead to \p VPBB, return the OR of all edges
+ /// or an equivalent block in-mask.
+ VPValue *createMaskDisjunction(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+
DenseMap<const VPBasicBlock *, VPBasicBlock::iterator> InsertPoints;
public:
- VPPredicator(VPlan &Plan) : VPDT(Plan), VPPDT(Plan) {}
+ VPPredicator(VPlan &Plan) : VPDT(Plan), VPPDT(Plan), VPPDF(VPPDT) {}
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(const VPBasicBlock *VPBB) const {
@@ -230,6 +242,115 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
setEdgeMask(Src, DefaultDst, DefaultMask);
}
+// Compute the "furthest up" set of edges for each incoming value of a phi.
+//
+// Start by keeping track of what edges lead to which value. Then see if any
+// node has the same value for all outgoing edges. If so then propagate that
+// value up to every node it postdominates.
+MapVector<VPPredicator::EdgeTy, VPValue *>
+VPPredicator::computeBlendEdges(VPPhi *Phi) {
+ MapVector<EdgeTy, VPValue *> Edges;
+
+ // Mark the given edge as providing the value \p V.
+ auto AddEdge = [&Edges](const VPBlockBase *From, const VPBlockBase *To,
+ VPValue *V) {
+ EdgeTy Edge = {cast<VPBasicBlock>(From), cast<VPBasicBlock>(To)};
+ assert((!Edges.contains(Edge) || Edges.lookup(Edge) == V) &&
+ "Clobbering an edge?");
+ Edges[Edge] = V;
+ };
+
+ for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
+ AddEdge(InVPBB, Phi->getParent(), InVal);
+
+ // The root phi must postdominate every incoming block. Also don't touch
+ // phis in a reduction chain since they need to be in a specific structure
+ // for handle*Reductions.
+ for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
+ if (!VPPDT.dominates(Phi->getParent(), InVPBB) ||
+ isa<VPReductionPHIRecipe>(InVal))
+ return Edges;
+
+ // Given a list of edges, check if they all have the same value and return it.
+ auto GetAllEqual = [&Edges](ArrayRef<EdgeTy> OutEdges) -> VPValue * {
+ VPValue *Common = nullptr;
+ for (EdgeTy E : OutEdges) {
+ VPValue *V = Edges.lookup(E);
+ if (!V)
+ return nullptr;
+ if (match(V, m_Poison()))
+ continue;
+ if (!Common)
+ Common = V;
+ else if (Common != V)
+ return nullptr;
+ }
+ return Common;
+ };
+
+ SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks());
+ while (!Worklist.empty()) {
+ auto *VPBB = cast<VPBasicBlock>(Worklist.pop_back_val());
+
+ // Check that all outgoing edges from VPBB have the same value.
+ SmallVector<EdgeTy> OutEdges;
+ for (const VPBlockBase *Succ : VPBB->getSuccessors())
+ OutEdges.emplace_back(VPBB, cast<VPBasicBlock>(Succ));
+ VPValue *Common = GetAllEqual(OutEdges);
+ if (!Common)
+ continue;
+
+ // They have the same value: we can move the edges up
+ for (EdgeTy Edge : OutEdges)
+ Edges.erase(Edge);
+
+ // Peek through phis that are postdominated by VPBB
+ if (auto *Phi = dyn_cast<VPPhi>(Common))
+ if (VPPDT.dominates(VPBB, Phi->getParent())) {
+ for (auto [InV, InVPBB] : Phi->incoming_values_and_blocks()) {
+ AddEdge(InVPBB, Phi->getParent(), InV);
+ Worklist.insert(InVPBB);
+ }
+ continue;
+ }
+
+ // Iterate up through the post dominance frontier
+ for (const VPBlockBase *Frontier : VPPDF.find(VPBB)->second) {
+ for (const VPBlockBase *FrontierSucc : Frontier->getSuccessors())
+ if (VPPDT.dominates(VPBB, FrontierSucc))
+ AddEdge(Frontier, FrontierSucc, Common);
+ Worklist.insert(cast<VPBasicBlock>(Frontier));
+ }
+ }
+
+ return Edges;
+}
+
+VPValue *VPPredicator::createMaskDisjunction(ArrayRef<EdgeTy> Edges,
+ VPBasicBlock *VPBB) {
+ auto Dsts = map_range(Edges, [](auto E) { return E.second; });
+ const VPBasicBlock *PostDom = *Dsts.begin();
+ for (const VPBasicBlock *VPBB : drop_begin(Dsts))
+ PostDom =
+ cast<VPBasicBlock>(VPPDT.findNearestCommonDominator(PostDom, VPBB));
+ assert(VPPDT.dominates(VPBB, PostDom) && "Edges don't postdominate VPBB");
+ if (PostDom != VPBB)
+ return getBlockInMask(PostDom);
+
+ VPValue *Mask = nullptr;
+ for (auto [Src, Dst] : Edges) {
+ VPValue *EdgeMask;
+ {
+ VPBuilder::InsertPointGuard Guard(Builder);
+ Builder.setInsertPoint(const_cast<VPBasicBlock *>(Dst),
+ InsertPoints[Dst]);
+ EdgeMask = createEdgeMask(Src, Dst);
+ }
+ Mask = Mask ? Builder.createOr(Mask, EdgeMask) : EdgeMask;
+ }
+ return Mask;
+}
+
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
Builder.setInsertPoint(VPBB, InsertPoints[VPBB]);
SmallVector<VPPhi *> Phis;
@@ -252,10 +373,30 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
continue;
}
+ MapVector<VPValue *, SmallVector<EdgeTy>> InValEdgesMap;
+ for (auto [Edge, Val] : computeBlendEdges(PhiR))
+ InValEdgesMap[Val].push_back(Edge);
+ auto InValEdges = InValEdgesMap.takeVector();
+
+ if (InValEdges.size() == 1) {
+ PhiR->replaceAllUsesWith(InValEdges[0].first);
+ PhiR->eraseFromParent();
+ continue;
+ }
+
+ // Sort the incoming value order to match PhiR as much as possible.
+ llvm::stable_sort(InValEdges, [&PhiR](auto &L, auto &R) {
+ auto InVs = PhiR->incoming_values();
+ return std::distance(InVs.begin(), find(InVs, L.first)) <
+ std::distance(InVs.begin(), find(InVs, R.first));
+ });
+
SmallVector<VPValue *, 2> OperandsWithMask;
- for (const auto &[InVPV, InVPBB] : PhiR->incoming_values_and_blocks()) {
+ for (const auto &[InVPV, Edges] : InValEdges) {
+ if (match(InVPV, m_Poison()))
+ continue;
OperandsWithMask.push_back(InVPV);
- OperandsWithMask.push_back(createEdgeMask(InVPBB, VPBB));
+ OperandsWithMask.push_back(createMaskDisjunction(Edges, VPBB));
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
auto *Blend =
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll
index 674ab702be327..4d388a3be84e4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator-early-exit.ll
@@ -46,15 +46,15 @@ define void @multi_early_exit_predicated_nested(ptr %p1, ptr %p2, i1 %c1, i1 %c2
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = not ir<%c1>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = or vp<[[VP11]]>, vp<[[VP12]]>
-; CHECK-NEXT: BLEND ir<%phi1.join2> = ir<1>/vp<[[VP8]]> ir<1>/vp<[[VP10]]> ir<0>/vp<[[VP12]]>
-; CHECK-NEXT: BLEND ir<%phi2.join2> = ir<1>/vp<[[VP8]]> ir<0>/vp<[[VP10]]> ir<0>/vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP10]]>
+; CHECK-NEXT: BLEND ir<%phi1.join2> = ir<1>/vp<[[VP14]]> ir<0>/vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = or vp<[[VP10]]>, vp<[[VP12]]>
+; CHECK-NEXT: BLEND ir<%phi2.join2> = ir<1>/vp<[[VP8]]> ir<0>/vp<[[VP15]]>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%ee2>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and ir<%c1>, ir<%ee1>
-; CHECK-NEXT: BLEND ir<%phi1> = ir<%phi1.join2>/vp<[[VP13]]> ir<1>/vp<[[VP14]]> ir<1>/vp<[[VP15]]>
-; CHECK-NEXT: BLEND ir<%phi2> = ir<%phi2.join2>/vp<[[VP13]]> ir<1>/vp<[[VP14]]> ir<poison>/vp<[[VP15]]>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<1>/ir<%c1> ir<0>/vp<[[VP13]]>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<1>/vp<[[VP6]]> ir<0>/vp<[[VP13]]>
; CHECK-NEXT: EMIT ir<%gep1> = getelementptr ir<%p1>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi1>, ir<%gep1>
; CHECK-NEXT: EMIT ir<%gep2> = getelementptr ir<%p2>, ir<%iv>
@@ -151,10 +151,8 @@ define void @multi_early_exit_predicated_not_nested(ptr %p1, ptr %p2, i1 %c1, i1
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%ee2>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and ir<%c1>, ir<%ee1>
-; CHECK-NEXT: BLEND ir<%phi1> = ir<%phi.join1>/vp<[[VP13]]> ir<%phi.join1>/vp<[[VP14]]> ir<1>/vp<[[VP15]]>
-; CHECK-NEXT: BLEND ir<%phi2> = ir<%phi.join2>/vp<[[VP13]]> ir<1>/vp<[[VP14]]> ir<poison>/vp<[[VP15]]>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<1>/ir<%c1> ir<0>/vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<1>/vp<[[VP8]]> ir<0>/vp<[[VP13]]>
; CHECK-NEXT: EMIT ir<%gep1> = getelementptr ir<%p1>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi1>, ir<%gep1>
; CHECK-NEXT: EMIT ir<%gep2> = getelementptr ir<%p2>, ir<%iv>
@@ -262,14 +260,10 @@ define void @four_exits_2x2_diamond(ptr %p1, ptr %p2, ptr %p3, ptr %p4, i1 %c1,
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%ee4>
-; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP12]]>, ir<%ee3>
-; CHECK-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%ee2>
-; CHECK-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and ir<%c1>, ir<%ee1>
-; CHECK-NEXT: BLEND ir<%phi1> = ir<%phi1.join1>/vp<[[VP17]]> ir<%phi1.join1>/vp<[[VP18]]> ir<%phi1.join1>/vp<[[VP19]]> ir<0>/vp<[[VP20]]> ir<1>/vp<[[VP21]]>
-; CHECK-NEXT: BLEND ir<%phi2> = ir<%phi2.join1>/vp<[[VP17]]> ir<%phi2.join1>/vp<[[VP18]]> ir<%phi2.join1>/vp<[[VP19]]> ir<1>/vp<[[VP20]]> ir<poison>/vp<[[VP21]]>
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%phi3.join2>/vp<[[VP17]]> ir<0>/vp<[[VP18]]> ir<1>/vp<[[VP19]]> ir<poison>/vp<[[VP20]]> ir<poison>/vp<[[VP21]]>
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%phi4.join2>/vp<[[VP17]]> ir<1>/vp<[[VP18]]> ir<poison>/vp<[[VP19]]> ir<poison>/vp<[[VP20]]> ir<poison>/vp<[[VP21]]>
+; CHECK-NEXT: BLEND ir<%phi1> = ir<0>/vp<[[VP4]]> ir<1>/ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<1>/vp<[[VP4]]> ir<0>/ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<0>/vp<[[VP11]]> ir<1>/vp<[[VP12]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP11]]> ir<0>/vp<[[VP12]]>
; CHECK-NEXT: EMIT ir<%gep1> = getelementptr ir<%p1>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi1>, ir<%gep1>
; CHECK-NEXT: EMIT ir<%gep2> = getelementptr ir<%p2>, ir<%iv>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 2e955de703cdf..76549d9b08ccf 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -309,7 +309,7 @@ define void @switch(ptr %a) {
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not vp<[[VP12]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP11]]>
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add2>/vp<[[VP5]]> ir<%add1>/vp<[[VP11]]> ir<%add1>/vp<[[VP11]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add2>/vp<[[VP5]]> ir<%add1>/vp<[[VP11]]>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP15]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
index 96775e0ff082e..d94b1d606e738 100644
--- a/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
@@ -545,8 +545,7 @@ define void @switch_unconditional_duplicate_target(ptr %start, ptr %dest) {
; IC1-NEXT: [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP0]], i64 0
; IC1-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META6:![0-9]+]]
; IC1-NEXT: [[TMP5:%.*]] = icmp ult <2 x i32> [[WIDE_LOAD]], splat (i32 10)
-; IC1-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP5]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP0]]
-; IC1-NEXT: [[PREDPHI2:%.*]] = select <2 x i1> [[TMP5]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[PREDPHI]]
+; IC1-NEXT: [[PREDPHI2:%.*]] = select <2 x i1> [[TMP5]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP0]]
; IC1-NEXT: [[TMP1:%.*]] = extractelement <2 x ptr> [[PREDPHI2]], i64 0
; IC1-NEXT: [[TMP2:%.*]] = extractelement <2 x ptr> [[PREDPHI2]], i64 1
; IC1-NEXT: store i32 0, ptr [[TMP1]], align 4
@@ -605,12 +604,10 @@ define void @switch_unconditional_duplicate_target(ptr %start, ptr %dest) {
; IC2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4, !alias.scope [[META6]]
; IC2-NEXT: [[TMP9:%.*]] = icmp ult <2 x i32> [[WIDE_LOAD]], splat (i32 10)
; IC2-NEXT: [[TMP10:%.*]] = icmp ult <2 x i32> [[WIDE_LOAD2]], splat (i32 10)
-; IC2-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP9]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP0]]
-; IC2-NEXT: [[PREDPHI2:%.*]] = select <2 x i1> [[TMP9]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[PREDPHI]]
+; IC2-NEXT: [[PREDPHI2:%.*]] = select <2 x i1> [[TMP9]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP0]]
; IC2-NEXT: [[TMP2:%.*]] = extractelement <2 x ptr> [[PREDPHI2]], i64 0
; IC2-NEXT: [[TMP3:%.*]] = extractelement <2 x ptr> [[PREDPHI2]], i64 1
-; IC2-NEXT: [[PREDPHI5:%.*]] = select <2 x i1> [[TMP10]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP1]]
-; IC2-NEXT: [[PREDPHI4:%.*]] = select <2 x i1> [[TMP10]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[PREDPHI5]]
+; IC2-NEXT: [[PREDPHI4:%.*]] = select <2 x i1> [[TMP10]], <2 x ptr> [[BROADCAST_SPLAT]], <2 x ptr> [[TMP1]]
; IC2-NEXT: [[TMP4:%.*]] = extractelement <2 x ptr> [[PREDPHI4]], i64 0
; IC2-NEXT: [[TMP5:%.*]] = extractelement <2 x ptr> [[PREDPHI4]], i64 1
; IC2-NEXT: store i32 0, ptr [[TMP2]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
index c0364c6fc5032..2c5459d472328 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
@@ -667,16 +667,14 @@ define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i1> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP4]], [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = and <4 x i1> [[TMP10]], [[TMP3]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
-; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP3]], <4 x float> [[PREDPHI2]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[TMP7]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index d9d30f8f3e0a5..2df4adc7802ff 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -1098,9 +1098,10 @@ define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
-; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP3]], <4 x float> [[PREDPHI2]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP14:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i1> [[TMP11]], [[TMP14]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP15]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -1147,12 +1148,14 @@ define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], [[WIDE_LOAD2]]
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = select <4 x i1> [[TMP21]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP20]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP13]]
-; CHECK-INTERLEAVED-NEXT: [[PREDPHI5:%.*]] = select <4 x i1> [[TMP15]], <4 x float> [[TMP17]], <4 x float> [[PREDPHI]]
-; CHECK-INTERLEAVED-NEXT: [[PREDPHI6]] = select <4 x i1> [[TMP5]], <4 x float> [[PREDPHI5]], <4 x float> [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP22]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP14]]
-; CHECK-INTERLEAVED-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP16]], <4 x float> [[TMP18]], <4 x float> [[PREDPHI7]]
-; CHECK-INTERLEAVED-NEXT: [[PREDPHI9]] = select <4 x i1> [[TMP6]], <4 x float> [[PREDPHI8]], <4 x float> [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = or <4 x i1> [[TMP20]], [[TMP27]]
+; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = or <4 x i1> [[TMP22]], [[TMP28]]
+; CHECK-INTERLEAVED-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP25]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP13]]
+; CHECK-INTERLEAVED-NEXT: [[PREDPHI6]] = select <4 x i1> [[TMP15]], <4 x float> [[TMP17]], <4 x float> [[PREDPHI]]
+; CHECK-INTERLEAVED-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP26]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP14]]
+; CHECK-INTERLEAVED-NEXT: [[PREDPHI9]] = select <4 x i1> [[TMP16]], <4 x float> [[TMP18]], <4 x float> [[PREDPHI7]]
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction.ll b/llvm/test/Transforms/LoopVectorize/reduction.ll
index c683ef897715a..12ddde4698ead 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction.ll
@@ -761,16 +761,14 @@ define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
-; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT: [[TMP6:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i1> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP4]], [[TMP6]]
; CHECK-NEXT: [[TMP11:%.*]] = and <4 x i1> [[TMP10]], [[TMP3]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
-; CHECK-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
-; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP3]], <4 x float> [[PREDPHI2]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[TMP7]], <4 x float> [[VEC_PHI]]
+; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
More information about the llvm-commits
mailing list