[llvm] [VPlan] Insert VPBlendRecipes in post order. NFC (PR #201782)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 00:33:20 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/201782
>From f16c94b767636408219f3b54a9d45c50774d79b7 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 8 Jun 2026 14:57:32 +0800
Subject: [PATCH 1/4] Precommit test
---
.../LoopVectorize/VPlan/predicator.ll | 108 ++++++++++++++++++
1 file changed, 108 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 2e955de703cdf..e8000cc8a25c3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -638,3 +638,111 @@ bb3:
exit:
ret void
}
+
+define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'blend_chain_non_trivial'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%lb> = load ir<%b>
+; CHECK-NEXT: EMIT ir<%v1> = add ir<%iv>, ir<%lb>
+; CHECK-NEXT: EMIT ir<%v2> = mul ir<%iv>, ir<3>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): if.a
+; CHECK-EMPTY:
+; CHECK-NEXT: if.a:
+; CHECK-NEXT: EMIT ir<%ca> = icmp sle ir<%iv>, ir<8>, ir<%c0>
+; CHECK-NEXT: Successor(s): if.a.inner
+; CHECK-EMPTY:
+; CHECK-NEXT: if.a.inner:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%c0>, ir<%ca>
+; CHECK-NEXT: Successor(s): merge.a.inner
+; CHECK-EMPTY:
+; CHECK-NEXT: merge.a.inner:
+; CHECK-NEXT: Successor(s): merge.a
+; CHECK-EMPTY:
+; CHECK-NEXT: merge.a:
+; CHECK-NEXT: EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): if.b
+; CHECK-EMPTY:
+; CHECK-NEXT: if.b:
+; CHECK-NEXT: EMIT ir<%cb> = icmp sle ir<%iv>, ir<16>, ir<%d0>
+; CHECK-NEXT: Successor(s): if.b.inner
+; CHECK-EMPTY:
+; CHECK-NEXT: if.b.inner:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
+; CHECK-NEXT: Successor(s): merge.b.inner
+; CHECK-EMPTY:
+; CHECK-NEXT: merge.b.inner:
+; CHECK-NEXT: Successor(s): merge.b
+; CHECK-EMPTY:
+; CHECK-NEXT: merge.b:
+; CHECK-NEXT: EMIT ir<%sum> = add ir<%v1>, ir<%v2>
+; CHECK-NEXT: EMIT store ir<%sum>, ir<%gep>
+; CHECK-NEXT: Successor(s): loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %lb = load i64, ptr %b
+ %v1 = add i64 %iv, %lb
+ %v2 = mul i64 %iv, 3
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ %c0 = icmp sle i64 %iv, 0
+ br i1 %c0, label %if.a, label %merge.a
+
+if.a:
+ %ca = icmp sle i64 %iv, 8
+ br i1 %ca, label %if.a.inner, label %merge.a.inner
+
+if.a.inner:
+ br label %merge.a.inner
+
+merge.a.inner:
+ %blend.a.inner = phi i64 [ %v1, %if.a ], [ %v1, %if.a.inner ]
+ br label %merge.a
+
+merge.a:
+ %blend.a = phi i64 [ %v1, %loop.header ], [ %blend.a.inner, %merge.a.inner ]
+ %d0 = icmp sgt i64 %iv, 0
+ br i1 %d0, label %if.b, label %merge.b
+
+if.b:
+ %cb = icmp sle i64 %iv, 16
+ br i1 %cb, label %if.b.inner, label %merge.b.inner
+
+if.b.inner:
+ br label %merge.b.inner
+
+merge.b.inner:
+ %blend.b.inner = phi i64 [ %v2, %if.b ], [ %v2, %if.b.inner ]
+ br label %merge.b
+
+merge.b:
+ %blend.b = phi i64 [ %v2, %merge.a ], [ %blend.b.inner, %merge.b.inner ]
+ %sum = add i64 %blend.a, %blend.b
+ store i64 %sum, ptr %gep
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret void
+}
>From fd134c39c732cd16fa7f30ac37b06703aafbc0a3 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 2 Jun 2026 18:15:25 +0800
Subject: [PATCH 2/4] Insert blends in post order
---
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp | 15 ++++++++++++---
.../Transforms/LoopVectorize/VPlan/predicator.ll | 8 ++++++--
2 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 2717b80e2eeaa..b01fef556f8ed 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -69,6 +69,8 @@ class VPPredicator {
return EdgeMaskCache[{Src, Dst}] = Mask;
}
+ DenseMap<const VPBasicBlock *, VPBasicBlock::iterator> InsertPoints;
+
public:
VPPredicator(VPlan &Plan) : VPDT(Plan), VPPDT(Plan) {}
@@ -136,6 +138,10 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// Start inserting after the block's phis, which be replaced by blends later.
Builder.setInsertPoint(VPBB, VPBB->getFirstNonPhi());
+ // Keep track of where in VPBB we are inserting the masks into.
+ scope_exit UpdateInsertPoint(
+ [this, &VPBB]() { InsertPoints[VPBB] = Builder.getInsertPoint(); });
+
// Reuse the mask of the immediate dominator if the VPBB post-dominates the
// immediate dominator.
auto *IDom = VPDT.getNode(VPBB)->getIDom();
@@ -225,6 +231,7 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
}
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
+ Builder.setInsertPoint(VPBB, InsertPoints[VPBB]);
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
Phis.push_back(cast<VPPhi>(&R));
@@ -276,10 +283,8 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
// Introduce the mask for VPBB, which may introduce needed edge masks, and
// convert all phi recipes of VPBB to blend recipes unless VPBB is the
// header.
- if (VPBB != Header) {
+ if (VPBB != Header)
Predicator.createBlockInMask(VPBB);
- Predicator.convertPhisToBlends(VPBB);
- }
VPValue *BlockMask = Predicator.getBlockInMask(VPBB);
if (!BlockMask)
@@ -292,6 +297,10 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
}
}
+ for (VPBlockBase *VPB : reverse(RPOT))
+ if (VPB != Header)
+ Predicator.convertPhisToBlends(cast<VPBasicBlock>(VPB));
+
// Linearize the blocks of the loop into one serial chain.
VPBlockBase *PrevVPBB = nullptr;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index e8000cc8a25c3..6f33d05b044e6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -665,6 +665,8 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): merge.a
; CHECK-EMPTY:
; CHECK-NEXT: merge.a:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: BLEND ir<%blend.a> = ir<%v1>/ir<%c0> ir<%v1>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): if.b
; CHECK-EMPTY:
@@ -673,14 +675,16 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): if.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: if.b.inner:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
; CHECK-NEXT: Successor(s): merge.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: merge.b.inner:
; CHECK-NEXT: Successor(s): merge.b
; CHECK-EMPTY:
; CHECK-NEXT: merge.b:
-; CHECK-NEXT: EMIT ir<%sum> = add ir<%v1>, ir<%v2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%d0>
+; CHECK-NEXT: BLEND ir<%blend.b> = ir<%v2>/ir<%d0> ir<%v2>/vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%sum> = add ir<%blend.a>, ir<%blend.b>
; CHECK-NEXT: EMIT store ir<%sum>, ir<%gep>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
>From 6b109a45eb763a1d6398aae132563ea14593c733 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 5 Jun 2026 18:31:56 +0800
Subject: [PATCH 3/4] Assert InsertPoints isn't clobbered
---
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index b01fef556f8ed..d8f042950e73a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -139,8 +139,10 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
Builder.setInsertPoint(VPBB, VPBB->getFirstNonPhi());
// Keep track of where in VPBB we are inserting the masks into.
- scope_exit UpdateInsertPoint(
- [this, &VPBB]() { InsertPoints[VPBB] = Builder.getInsertPoint(); });
+ scope_exit UpdateInsertPoint([this, &VPBB]() {
+ assert(!InsertPoints.contains(VPBB) && "InsertPoint clobbered?");
+ InsertPoints[VPBB] = Builder.getInsertPoint();
+ });
// Reuse the mask of the immediate dominator if the VPBB post-dominates the
// immediate dominator.
>From 73427ee2b2dec03dd65f9f6c65e4bb0d31a969f6 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 8 Jun 2026 15:13:55 +0800
Subject: [PATCH 4/4] Remove InsertPoints map
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index d8f042950e73a..2878a619119de 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -69,7 +69,14 @@ class VPPredicator {
return EdgeMaskCache[{Src, Dst}] = Mask;
}
- DenseMap<const VPBasicBlock *, VPBasicBlock::iterator> InsertPoints;
+ /// Returns where to insert new masks in \p VPBB.
+ VPBasicBlock::iterator getMaskInsertPoint(VPBasicBlock *VPBB) {
+ if (VPValue *Mask = getBlockInMask(VPBB))
+ if (VPRecipeBase *MaskR = Mask->getDefiningRecipe())
+ if (MaskR->getParent() == VPBB) // In-mask may be the IDom's.
+ return std::next(MaskR->getIterator());
+ return VPBB->getFirstNonPhi();
+ }
public:
VPPredicator(VPlan &Plan) : VPDT(Plan), VPPDT(Plan) {}
@@ -138,12 +145,6 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// Start inserting after the block's phis, which be replaced by blends later.
Builder.setInsertPoint(VPBB, VPBB->getFirstNonPhi());
- // Keep track of where in VPBB we are inserting the masks into.
- scope_exit UpdateInsertPoint([this, &VPBB]() {
- assert(!InsertPoints.contains(VPBB) && "InsertPoint clobbered?");
- InsertPoints[VPBB] = Builder.getInsertPoint();
- });
-
// Reuse the mask of the immediate dominator if the VPBB post-dominates the
// immediate dominator.
auto *IDom = VPDT.getNode(VPBB)->getIDom();
@@ -233,7 +234,8 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
}
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
- Builder.setInsertPoint(VPBB, InsertPoints[VPBB]);
+ Builder.setInsertPoint(VPBB, getMaskInsertPoint(VPBB));
+
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
Phis.push_back(cast<VPPhi>(&R));
More information about the llvm-commits
mailing list