[llvm] [VPlan] Peek through nested phi incoming values in computeBlendMasks (PR #203164)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 05:29:27 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/203164
>From 9318bfed7adc3d91a9cd74ef950c6703eb6cb4fd Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 9 Jun 2026 20:10:59 +0800
Subject: [PATCH 1/5] Precommit tests
---
.../LoopVectorize/VPlan/predicator.ll | 162 ++++++++++++++++++
.../Transforms/LoopVectorize/predicator.ll | 149 ++++++++++++++++
2 files changed, 311 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 483ab8ad94b50..e51eb0001e77c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -959,3 +959,165 @@ latch:
exit:
ret void
}
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_multi_use'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ store i32 %phi4, ptr %gep
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 57414ae62c341..cfbb54ad9fac0 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -292,3 +292,152 @@ latch:
exit:
ret void
}
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[BROADCAST_SPLAT6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_multi_use(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT7]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[PREDPHI]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ store i32 %phi4, ptr %gep
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
>From 1fa35284085ce96d65b5a1da6b94c9fefe68bc2b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 9 Jun 2026 20:12:38 +0800
Subject: [PATCH 2/5] Peek through inner phi's nested values
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 21 +++++++++++++++++++
.../LoopVectorize/VPlan/predicator.ll | 10 +++------
.../Transforms/LoopVectorize/predicator.ll | 10 +--------
.../Transforms/LoopVectorize/uniform-blend.ll | 15 +++----------
4 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 3f35f14e876f2..89e7ec74f5e3d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -275,6 +275,10 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
AddEdge(InVPBB, Phi->getParent(), InVal);
+ // Don't optimize any reduction chains for now.
+ if (any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>))
+ return Edges;
+
SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks());
while (!Worklist.empty()) {
auto *VPBB = cast<VPBasicBlock>(Worklist.pop_back_val());
@@ -293,6 +297,17 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (EdgeTy Edge : OutEdges)
Edges.erase(Edge);
+ // If the value is a phi postdominated by VPBB, then look through the inner
+ // incoming values instead of propagating the phi.
+ if (auto *Phi = dyn_cast<VPPhi>(Common))
+ if (Phi->hasOneUse() && VPPDT.dominates(VPBB, Phi->getParent())) {
+ for (auto [InV, InVPBB] : Phi->incoming_values_and_blocks()) {
+ AddEdge(InVPBB, Phi->getParent(), InV);
+ Worklist.insert(InVPBB);
+ }
+ continue;
+ }
+
// Iterate up through the post dominance frontier.
assert(VPPDF.find(VPBB) != VPPDF.end() &&
"VPBB must have a post-dominance frontier entry");
@@ -373,6 +388,12 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
InValEdgesMap[Val].push_back(Edge);
auto InValEdges = InValEdgesMap.takeVector();
+ if (InValEdges.size() == 1) {
+ PhiR->replaceAllUsesWith(InValEdges[0].first);
+ PhiR->eraseFromParent();
+ continue;
+ }
+
// Sort the incoming value order to match PhiR as much as possible.
llvm::stable_sort(InValEdges, [&PhiR](auto &L, auto &R) {
auto InVs = PhiR->incoming_values();
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index e51eb0001e77c..3941d865093e0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -665,8 +665,6 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): merge.a
; CHECK-EMPTY:
; CHECK-NEXT: merge.a:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: BLEND ir<%blend.a> = ir<%v1>/ir<%c0> ir<%v1>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): if.b
; CHECK-EMPTY:
@@ -675,16 +673,14 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): if.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: if.b.inner:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
; CHECK-NEXT: Successor(s): merge.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: merge.b.inner:
; CHECK-NEXT: Successor(s): merge.b
; CHECK-EMPTY:
; CHECK-NEXT: merge.b:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%d0>
-; CHECK-NEXT: BLEND ir<%blend.b> = ir<%v2>/ir<%d0> ir<%v2>/vp<[[VP7]]>
-; CHECK-NEXT: EMIT ir<%sum> = add ir<%blend.a>, ir<%blend.b>
+; CHECK-NEXT: EMIT ir<%sum> = add ir<%v1>, ir<%v2>
; CHECK-NEXT: EMIT store ir<%sum>, ir<%gep>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
@@ -988,7 +984,7 @@ define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
-; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]>
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%x>/ir<%c0> ir<%y>/vp<[[VP4]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index cfbb54ad9fac0..bce5d335b39df 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -303,15 +303,7 @@ define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[BROADCAST_SPLAT6]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 8d86c8ca2bd9d..a016a26d479ec 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -96,19 +96,10 @@ define void @blend_chain_iv(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP7]]
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP4]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP6]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP8]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP8]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From f391e26ecec4c638dc8b46938ce1811c490fb1fc Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 3 Aug 2026 13:16:56 +0800
Subject: [PATCH 3/5] Fix inserting blends before incoming edge masks
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 15 +++-
.../LoopVectorize/VPlan/predicator.ll | 72 +++++++++++++++++++
.../Transforms/LoopVectorize/predicator.ll | 64 +++++++++++++++++
3 files changed, 150 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 89e7ec74f5e3d..3523460f6068c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -81,6 +81,19 @@ class VPPredicator {
return VPBB->getFirstNonPhi();
}
+ /// Returns where to insert new blends in \p VPBB.
+ VPBasicBlock::iterator getBlendInsertPoint(VPBasicBlock *VPBB) {
+ VPBasicBlock::iterator It = getMaskInsertPoint(VPBB);
+ // Insert after any incoming edge masks if they were already inserted.
+ for (VPBlockBase *Pred : VPBB->predecessors())
+ if (VPValue *EdgeMask = getEdgeMask(cast<VPBasicBlock>(Pred), VPBB))
+ if (VPRecipeBase *EdgeR = EdgeMask->getDefiningRecipe())
+ if (It != VPBB->end() && EdgeR->getParent() == VPBB &&
+ !VPDT.properlyDominates(EdgeR, &*It))
+ It = std::next(EdgeR->getIterator());
+ return It;
+ }
+
using EdgeTy = std::pair<const VPBasicBlock *, const VPBasicBlock *>;
/// Compute the set of edges that are "furthest up" in the CFG for each
@@ -361,7 +374,7 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
}
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
- Builder.setInsertPoint(VPBB, getMaskInsertPoint(VPBB));
+ Builder.setInsertPoint(VPBB, getBlendInsertPoint(VPBB));
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 3941d865093e0..01e07a4014cb5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -1117,3 +1117,75 @@ bb5:
exit:
ret void
}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_blend_insert_pos'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<2>/vp<[[VP6]]> ir<1>/vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%c0>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP8]]> ir<2>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / \ \
+; bb2 | /
+; \ / /
+; bb3 /
+; \ /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret i32 %phi4
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index bce5d335b39df..29f30b02ed85b 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -433,3 +433,67 @@ bb5:
exit:
ret void
}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @look_thru_phi_blend_insert_pos(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> splat (i32 2), <4 x i32> splat (i32 1)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+; bb0
+; / \
+; bb1 \
+; / \ \
+; bb2 | /
+; \ / /
+; bb3 /
+; \ /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret i32 %phi4
+}
>From 17eb3204e9bb5a089e112e68e40ed830ffde3047 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 3 Aug 2026 13:57:50 +0800
Subject: [PATCH 4/5] Relax reduction restriction to phis inserted for tail
folding
---
.../lib/Transforms/Vectorize/VPlanPredicator.cpp | 7 +++++--
.../LoopVectorize/select-cmp-blend-chain.ll | 16 +++++++++++++---
2 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 3523460f6068c..09237ab575f8c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -288,8 +288,11 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
AddEdge(InVPBB, Phi->getParent(), InVal);
- // Don't optimize any reduction chains for now.
- if (any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>))
+ // Don't optimize any phis used to mask tail values of reductions during tail
+ // folding until handleFindLastReductions can match the new form.
+ VPlan *Plan = Phi->getParent()->getPlan();
+ if (Phi->getParent() == Plan->getVectorLoopRegion()->getExiting() &&
+ any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>))
return Edges;
SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks());
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
index 8b12e09631246..ca6c1833eee83 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
@@ -94,15 +94,25 @@ define i32 @anyof_three_blend_chain(i1 %c0, i1 %c1, i1 %c2, i32 %n) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <4 x i32> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP0]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C2]], <4 x i1> [[VEC_PHI]], <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[C1]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI1:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP7]]
; CHECK-NEXT: [[PREDPHI2]] = select i1 [[C0]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
>From 73ffea8931ccfb537808c5d0254195b4aa695b28 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 4 Aug 2026 00:28:26 +0800
Subject: [PATCH 5/5] Early return on It == VPBB->end()
---
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 09237ab575f8c..5247bd000aa02 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -84,13 +84,18 @@ class VPPredicator {
/// Returns where to insert new blends in \p VPBB.
VPBasicBlock::iterator getBlendInsertPoint(VPBasicBlock *VPBB) {
VPBasicBlock::iterator It = getMaskInsertPoint(VPBB);
+ if (It == VPBB->end())
+ return It;
// Insert after any incoming edge masks if they were already inserted.
for (VPBlockBase *Pred : VPBB->predecessors())
if (VPValue *EdgeMask = getEdgeMask(cast<VPBasicBlock>(Pred), VPBB))
if (VPRecipeBase *EdgeR = EdgeMask->getDefiningRecipe())
- if (It != VPBB->end() && EdgeR->getParent() == VPBB &&
- !VPDT.properlyDominates(EdgeR, &*It))
+ if (EdgeR->getParent() == VPBB &&
+ !VPDT.properlyDominates(EdgeR, &*It)) {
It = std::next(EdgeR->getIterator());
+ if (It == VPBB->end())
+ return It;
+ }
return It;
}
More information about the llvm-commits
mailing list