[llvm-branch-commits] [llvm] [VPlan] Remove header masks whose tail lanes aren't used (PR #241747)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 12 02:16:17 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/241747
>From f60fdbfe57747b1751a439f81298231dba08c998 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 2 Nov 2026 10:13:41 +0800
Subject: [PATCH] [VPlan] Remove header masks whose tail lanes aren't used
Replace a use of header-mask && x with x if the user, transitively
through freeze, logical-and, not and mask.beforefirst(freeze ...), only
reaches masks that are and'ed with the header mask again. The header
mask is a prefix and lane i of mask.beforefirst only depends on lanes
0..i, so the lanes outside of the header mask are never used.
This avoids the i1 vp.merge that EVL tail folding needs for the inner
header mask, e.g. in the early exit mask.beforefirst(freeze(header-mask
&& c)) and in switch default edges.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 39 +++++++++++++++++++
.../LoopVectorize/RISCV/early-exit.ll | 6 +--
.../RISCV/early_exit_with_stores.ll | 6 +--
.../LoopVectorize/RISCV/zvl1024b-crash.ll | 3 +-
.../LoopVectorize/early-exit-tail-folding.ll | 6 +--
5 files changed, 46 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 201987855142a0..9b4a8afdebfa48 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1806,6 +1806,31 @@ void VPlanTransforms::simplifyReverses(VPlan &Plan) {
R.getVPSingleValue()->replaceAllUsesWith(X);
}
+/// Returns true if \p U doesn't use the lanes of its operand \p V outside of
+/// the header mask \p HM, so they can be replaced by any value, even poison.
+static bool ignoresTailLanes(VPUser *U, VPValue *V, VPValue *HM,
+ SmallPtrSetImpl<VPUser *> &Visited) {
+ if (match(U, m_LogicalAnd(m_Specific(HM), m_Specific(V))))
+ return true;
+ // Lane i of mask.beforefirst only depends on lanes 0..i of its operand, and
+ // the header mask is a prefix. It's poison if any lane is, so needs a freeze.
+ if (!match(U, m_CombineOr(m_Freeze(m_VPValue()),
+ m_LogicalAnd(m_VPValue(), m_VPValue()),
+ m_Not(m_VPValue()),
+ m_Intrinsic<Intrinsic::mask_beforefirst>(
+ m_Freeze(m_VPValue())))))
+ return false;
+ // Masks can share subexpressions, e.g. X && !(X && Y), so don't revisit users
+ // to avoid an exponential number of paths. The search stops at the first user
+ // that uses the tail lanes, so any user seen before must have ignored them.
+ if (!Visited.insert(U).second)
+ return true;
+ auto *R = cast<VPSingleDefRecipe>(U);
+ return all_of(R->users(), [&](VPUser *RU) {
+ return ignoresTailLanes(RU, R, HM, Visited);
+ });
+}
+
/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
/// header mask to be simplified further when tail folding, e.g. in
/// optimizeEVLMasks.
@@ -1831,6 +1856,20 @@ static void reassociateHeaderMask(VPlan &Plan) {
R->replaceAllUsesWith(
Builder.createLogicalAnd(HeaderMask, Builder.createLogicalAnd(X, Y)));
}
+
+ // Replace uses of headermask && x with x if they don't use the lanes outside
+ // of the header mask.
+ for (VPUser *U : to_vector(HeaderMask->users())) {
+ VPValue *X;
+ if (!match(U, m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(X))))
+ continue;
+ auto *And = cast<VPSingleDefRecipe>(U);
+ for (VPUser *AndU : to_vector(And->users())) {
+ SmallPtrSet<VPUser *, 8> Visited;
+ if (ignoresTailLanes(AndU, And, HeaderMask, Visited))
+ AndU->replaceUsesOfWith(And, X);
+ }
+ }
}
static std::optional<Instruction::BinaryOps>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
index 06afcc1d4e1c8b..deb68b431759b3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -416,15 +416,13 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500)
-; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP10]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
-; CHECK-NEXT: [[TMP12:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP9]], false
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP0]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index a566df5db77019..2dd29fe2ac8f70 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1278,8 +1278,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i1> @llvm.vp.merge.nxv8i1(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i1> [[TMP3]], <vscale x 8 x i1> zeroinitializer, i32 [[TMP0]])
-; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP6]]
@@ -1287,8 +1286,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2))
; CHECK-NEXT: call void @llvm.vp.store.nxv16i16.p0(<vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], i32 [[INTERLEAVE_EVL]])
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv8i1(i1 false, <vscale x 8 x i1> [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv8i1(i1 false, <vscale x 8 x i1> [[TMP4]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP9]], false
; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP0]] to i64
; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP11]], [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll
index 5dc622a791e212..df762cd3594c31 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/zvl1024b-crash.ll
@@ -26,13 +26,12 @@ define void @foo(i32 %0, ptr %p) vscale_range(16, 1024) {
; CHECK-NEXT: [[TMP15:%.*]] = or <vscale x 16 x i1> [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 16 x i1> [[TMP15]], [[TMP8]]
; CHECK-NEXT: [[TMP17:%.*]] = or <vscale x 16 x i1> [[TMP16]], [[TMP9]]
+; CHECK-NEXT: [[TMP20:%.*]] = xor <vscale x 16 x i1> [[TMP17]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[CURRENT_ITERATION_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 524, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
-; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.vp.merge.nxv16i1(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i1> [[TMP17]], <vscale x 16 x i1> zeroinitializer, i32 [[TMP18]])
-; CHECK-NEXT: [[TMP20:%.*]] = xor <vscale x 16 x i1> [[TMP19]], splat (i1 true)
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[P]], i64 [[CURRENT_ITERATION_IV]]
; CHECK-NEXT: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> zeroinitializer, ptr align 1 [[TMP23]], <vscale x 16 x i1> [[TMP20]], i32 [[TMP18]])
; CHECK-NEXT: [[TMP24:%.*]] = zext i32 [[TMP18]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index 74cd834636c7c3..59f00b2e219a96 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -286,16 +286,14 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 500)
-; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[TMP6]], <4 x i32> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], splat (i32 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
More information about the llvm-branch-commits
mailing list