[llvm] [VPlan] Introduce m_SelectLike and use to support 2-operand blends. (PR #194729)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 11:02:47 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/194729
>From b64b0cf00aa3948da8d369b0e7996d253a6a5827 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 28 Apr 2026 15:33:13 +0100
Subject: [PATCH 1/4] [VPlan] Introduce m_SelectLike and use to support
2-operand blends.
We should be able to treat 2-operand blends like select by most VPlan
code. Add a new m_SelectLike matcher and use in places that only use the
matcher to extract operands.
Overall this leads to a small number of improvements in RISV (~10 files
changed in a large IR corpus) and 2 loops changed on AArch64 with
tail-folding forced.
This could also avoid the need to perform manual blend -> select
conversion in https://github.com/llvm/llvm-project/pull/192987.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 13 +---
.../Vectorize/VPlanConstruction.cpp | 7 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 31 +++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 40 ++++++------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
.../LoopVectorize/AArch64/select-costs.ll | 4 +-
.../RISCV/tail-folding-complex-mask.ll | 3 +-
...-order-recurrence-sink-replicate-region.ll | 2 +-
.../LoopVectorize/epilog-iv-select-cmp.ll | 64 ++++++++++++++-----
9 files changed, 111 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8cef47b898843..80fc105f2969d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6979,17 +6979,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
Type *PhiTy = TypeInfo.inferScalarType(PhiR);
// Convert a VPBlendRecipe backedge to a select.
- if (auto *Blend = dyn_cast<VPBlendRecipe>(PhiR->getBackedgeValue())) {
- if (Blend->getNumIncomingValues() == 2 &&
- Blend->getMask(0) == HeaderMask) {
- auto *Sel = VPBuilder(Blend).createSelect(
- Blend->getMask(0), Blend->getIncomingValue(0),
- Blend->getIncomingValue(1), {}, "", *Blend);
- Blend->replaceAllUsesWith(Sel);
- Blend->eraseFromParent();
- }
- }
-
auto *OrigExitingVPV = PhiR->getBackedgeValue();
auto *NewExitingVPV = PhiR->getBackedgeValue();
@@ -6997,7 +6986,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *V;
if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
match(PhiR->getBackedgeValue(),
- m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
+ m_SelectLike(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
// We want code in the middle block to appear to execute on the location of
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 793eaf277f563..11728efe3dad5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1763,9 +1763,10 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
// If there's a header mask, the backedge select will not be the find-last
// select.
- if (HeaderMask && !match(BackedgeSelect,
- m_Select(m_Specific(HeaderMask),
- m_VPValue(CondSelect), m_Specific(PhiR))))
+ if (HeaderMask &&
+ !match(BackedgeSelect,
+ m_SelectLike(m_Specific(HeaderMask), m_VPValue(CondSelect),
+ m_Specific(PhiR))))
return false;
VPValue *Cond = nullptr, *Op1 = nullptr, *Op2 = nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index f8631549bc179..8e48db2b0ea5b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -756,6 +756,29 @@ inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
}
+/// Match a VPBlendRecipe with 2 incoming values ([M0, I0, M1, I1]) as
+/// select(M1, I1, I0). Also matches the normalized form, where M0 is dropped.
+template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
+ Op0_t MaskOp;
+ Op1_t TrueOp;
+ Op2_t FalseOp;
+
+ Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
+ : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
+
+ template <typename T> bool match(const T *Val) const {
+ auto *Blend = dyn_cast<VPBlendRecipe>(Val);
+ if (!Blend || Blend->getNumIncomingValues() != 2)
+ return false;
+ unsigned TrueIdx = Blend->isNormalized() ? 1 : 0;
+ unsigned FalseIdx = 1 - TrueIdx;
+ return MaskOp.match(Blend->getMask(TrueIdx)) &&
+ TrueOp.match(Blend->getIncomingValue(TrueIdx)) &&
+ FalseOp.match(Blend->getIncomingValue(FalseIdx));
+ }
+};
+
+/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
template <typename Op0_t, typename Op1_t, typename Op2_t>
inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
@@ -763,6 +786,14 @@ m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
{Op0, Op1, Op2});
}
+/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
+/// incoming values.
+template <typename Op0_t, typename Op1_t, typename Op2_t>
+inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
+ return m_CombineOr(m_Select(Op0, Op1, Op2),
+ Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
+}
+
template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d3df6471c9083..e1b964333f492 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3005,14 +3005,14 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
return new VPInterleaveEVLRecipe(*Interleave, EVL, Mask);
VPValue *LHS, *RHS;
- if (match(&CurRecipe,
- m_Select(m_Specific(HeaderMask), m_VPValue(LHS), m_VPValue(RHS))))
+ if (match(&CurRecipe, m_SelectLike(m_Specific(HeaderMask), m_VPValue(LHS),
+ m_VPValue(RHS))))
return new VPWidenIntrinsicRecipe(
Intrinsic::vp_merge, {Plan->getTrue(), LHS, RHS, &EVL},
TypeInfo.inferScalarType(LHS), {}, {}, DL);
- if (match(&CurRecipe, m_Select(m_RemoveMask(HeaderMask, Mask), m_VPValue(LHS),
- m_VPValue(RHS))))
+ if (match(&CurRecipe, m_SelectLike(m_RemoveMask(HeaderMask, Mask),
+ m_VPValue(LHS), m_VPValue(RHS))))
return new VPWidenIntrinsicRecipe(
Intrinsic::vp_merge, {Mask, LHS, RHS, &EVL},
TypeInfo.inferScalarType(LHS), {}, {}, DL);
@@ -3175,8 +3175,8 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
// in the vector loop, not the middle block, since EVL tail folding can have
// tail elements in the penultimate iteration.
assert(all_of(*Plan.getMiddleBlock(), [&Plan, HeaderMask](VPRecipeBase &R) {
- if (match(&R, m_ComputeReductionResult(m_Select(m_Specific(HeaderMask),
- m_VPValue(), m_VPValue()))))
+ if (match(&R, m_ComputeReductionResult(m_SelectLike(
+ m_Specific(HeaderMask), m_VPValue(), m_VPValue()))))
return R.getOperand(0)->getDefiningRecipe()->getRegion() ==
Plan.getVectorLoopRegion();
return true;
@@ -5708,20 +5708,20 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPValue *BackedgeVal = PhiR->getBackedgeValue();
auto *FindLastSelect = cast<VPSingleDefRecipe>(BackedgeVal);
if (HeaderMask &&
- !match(BackedgeVal,
- m_Select(m_Specific(HeaderMask),
- m_VPSingleDefRecipe(FindLastSelect), m_Specific(PhiR))))
+ !match(BackedgeVal, m_SelectLike(m_Specific(HeaderMask),
+ m_VPSingleDefRecipe(FindLastSelect),
+ m_Specific(PhiR))))
continue;
// Get the find-last expression from the find-last select of the reduction
// phi. The find-last select should be a select between the phi and the
// find-last expression.
VPValue *Cond, *FindLastExpression;
- if (!match(FindLastSelect, m_Select(m_VPValue(Cond), m_Specific(PhiR),
- m_VPValue(FindLastExpression))) &&
+ if (!match(FindLastSelect, m_SelectLike(m_VPValue(Cond), m_Specific(PhiR),
+ m_VPValue(FindLastExpression))) &&
!match(FindLastSelect,
- m_Select(m_VPValue(Cond), m_VPValue(FindLastExpression),
- m_Specific(PhiR))))
+ m_SelectLike(m_VPValue(Cond), m_VPValue(FindLastExpression),
+ m_Specific(PhiR))))
continue;
// Check if FindLastExpression is a simple expression of a widened IV. If
@@ -6075,9 +6075,9 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// Check if WidenRecipe is the final result of the reduction. If so look
// through selects for predicated reductions.
VPValue *Cond = nullptr;
- VPValue *ExitValue = cast_or_null<VPInstruction>(vputils::findUserOf(
- WidenRecipe,
- m_Select(m_VPValue(Cond), m_Specific(WidenRecipe), m_Specific(RdxPhi))));
+ VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(vputils::findUserOf(
+ WidenRecipe, m_SelectLike(m_VPValue(Cond), m_Specific(WidenRecipe),
+ m_Specific(RdxPhi))));
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue;
assert((!ExitValue || IsLastInChain) &&
@@ -6314,7 +6314,8 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
- match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
+ match(ExitValue,
+ m_SelectLike(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
VPTypeAnalysis &TypeInfo = CostCtx.Types;
SmallVector<VPPartialReductionChain> Chain;
@@ -6449,8 +6450,9 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
match(R, m_ComputeReductionResult(
m_Specific(Chain.ReductionBinOp))) ||
- match(R, m_Select(m_VPValue(), m_Specific(Chain.ReductionBinOp),
- m_Specific(RedPhiR)));
+ match(R,
+ m_SelectLike(m_VPValue(), m_Specific(Chain.ReductionBinOp),
+ m_Specific(RedPhiR)));
};
if (!all_of(Chain.ReductionBinOp->users(), UseIsValid)) {
Chains.clear();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 24ff4080d8a2b..02c4354b6e3d6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -802,7 +802,7 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
// Look through selects inserted for tail folding or predicated reductions.
VPRecipeBase *SelR = vputils::findUserOf(
- BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue()));
if (!SelR)
return nullptr;
return vputils::findUserOf<VPInstruction::ComputeReductionResult>(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
index add503acb9420..9c8bb9892c622 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
@@ -69,8 +69,8 @@ exit:
define i32 @select_vpinst_for_tail_folding(i8 %n) {
; CHECK: LV: Checking a loop in 'select_vpinst_for_tail_folding'
-; CHECK: Cost of 1 for VF 2: EMIT vp<{{.+}}> = select vp<{{.+}}>, ir<%red.next>, ir<%red>
-; CHECK: Cost of 1 for VF 4: EMIT vp<{{.+}}> = select vp<{{.+}}>, ir<%red.next>, ir<%red>
+; CHECK: Cost of 6 for VF 2: BLEND vp<{{.+}}> = ir<%red> ir<%red.next>/vp<{{.+}}>
+; CHECK: Cost of 12 for VF 4: BLEND vp<{{.+}}> = ir<%red> ir<%red.next>/vp<{{.+}}>
; CHECK: LV: Selecting VF: 4
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 7a189384ee264..a5a1c709251ec 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -30,12 +30,11 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC0]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], i32 [[TMP7]])
-; IF-EVL-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP7]])
; IF-EVL-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[VP_OP_LOAD]]
; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[SRC1]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD7]], [[PREDPHI]]
-; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[PREDPHI8:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[BROADCAST_SPLAT4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[PREDPHI8]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 280e3ef8fbb42..07116a9d16976 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -270,7 +270,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%and.red.next>, ir<%and.red>
+; CHECK-NEXT: BLEND vp<[[VP10:%[0-9]+]]> = ir<%and.red> ir<%and.red.next>/vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (and) vp<[[VP10]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
index 0a98a895f3223..4e13270c87499 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll
@@ -913,35 +913,69 @@ exit:
define i32 @predicated_iv_select(ptr %A) {
; CHECK-LABEL: define i32 @predicated_iv_select(
; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH1:.*]], label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <8 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH1]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <8 x i32> [[WIDE_LOAD]], splat (i32 -1)
; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[VEC_IND]], ptr align 4 [[TMP0]], <8 x i1> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
-; CHECK-NEXT: [[TMP5]] = select i1 [[TMP4]], <8 x i1> [[TMP1]], <8 x i1> [[TMP7]]
-; CHECK-NEXT: [[TMP6]] = select i1 [[TMP4]], <8 x i32> [[VEC_IND]], <8 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP1]], <8 x i32> [[VEC_IND]], <8 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8)
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1104
; CHECK-NEXT: br i1 [[TMP2]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
-; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v8i32(<8 x i32> [[TMP6]], <8 x i1> [[TMP5]], i32 -1)
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[PREDPHI]])
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP3]], -2147483648
+; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP3]], i32 -1
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH1]], !prof [[PROF3]]
+; CHECK: [[VEC_EPILOG_PH1]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 1104, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[VECTOR_PH]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[BC_MERGE_RDX]], -1
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[BC_MERGE_RDX]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH1]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[BROADCAST_SPLAT]], %[[VEC_EPILOG_PH1]] ], [ [[PREDPHI7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND5:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VEC_EPILOG_PH1]] ], [ [[VEC_IND_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX3]]
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD6]], splat (i32 -1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[VEC_IND5]], ptr align 4 [[TMP9]], <4 x i1> [[TMP10]])
+; CHECK-NEXT: [[PREDPHI7]] = select <4 x i1> [[TMP10]], <4 x i32> [[VEC_IND5]], <4 x i32> [[VEC_PHI4]]
+; CHECK-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX3]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT9]] = add <4 x i32> [[VEC_IND5]], splat (i32 4)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT8]], 1108
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[PREDPHI7]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i32 [[TMP12]], -2147483648
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 -1
+; CHECK-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1108, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 1104, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_ITER_CHECK]] ], [ -1, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1104, %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[TMP8]], %[[VEC_EPILOG_VECTOR_BODY]] ], [ [[LAST_1:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[LAST_1:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP15]], -1
@@ -954,9 +988,9 @@ define i32 @predicated_iv_select(ptr %A) {
; CHECK-NEXT: [[LAST_1]] = phi i32 [ [[T]], %[[LOOP_THEN]] ], [ [[RED]], %[[LOOP_HEADER]] ]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1111
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP30:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[LAST_1_LCSSA:%.*]] = phi i32 [ [[LAST_1]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[LAST_1_LCSSA:%.*]] = phi i32 [ [[LAST_1]], %[[LOOP_LATCH]] ], [ [[TMP5]], %[[VEC_EPILOG_PH]] ], [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[LAST_1_LCSSA]]
;
entry:
>From 33f89c76c49ac9b9c0628100cc87ed03d802a84c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 19 May 2026 15:26:33 +0100
Subject: [PATCH 2/4] !fixup remve m_SelectLike use from fixupVFUsersForEVL.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e1b964333f492..ed3d288599aa0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3175,8 +3175,8 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
// in the vector loop, not the middle block, since EVL tail folding can have
// tail elements in the penultimate iteration.
assert(all_of(*Plan.getMiddleBlock(), [&Plan, HeaderMask](VPRecipeBase &R) {
- if (match(&R, m_ComputeReductionResult(m_SelectLike(
- m_Specific(HeaderMask), m_VPValue(), m_VPValue()))))
+ if (match(&R, m_ComputeReductionResult(m_Select(m_Specific(HeaderMask),
+ m_VPValue(), m_VPValue()))))
return R.getOperand(0)->getDefiningRecipe()->getRegion() ==
Plan.getVectorLoopRegion();
return true;
>From 4fe8108c4bfac7f5f7381b03a4dfda9673b8e16b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 20 May 2026 15:28:15 +0100
Subject: [PATCH 3/4] !fixup remove some changes
---
.../Transforms/Vectorize/LoopVectorize.cpp | 13 +++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 20 +++++++++----------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
.../LoopVectorize/AArch64/select-costs.ll | 4 ++--
...-order-recurrence-sink-replicate-region.ll | 2 +-
5 files changed, 25 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 80fc105f2969d..8cef47b898843 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6979,6 +6979,17 @@ void LoopVectorizationPlanner::addReductionResultComputation(
Type *PhiTy = TypeInfo.inferScalarType(PhiR);
// Convert a VPBlendRecipe backedge to a select.
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(PhiR->getBackedgeValue())) {
+ if (Blend->getNumIncomingValues() == 2 &&
+ Blend->getMask(0) == HeaderMask) {
+ auto *Sel = VPBuilder(Blend).createSelect(
+ Blend->getMask(0), Blend->getIncomingValue(0),
+ Blend->getIncomingValue(1), {}, "", *Blend);
+ Blend->replaceAllUsesWith(Sel);
+ Blend->eraseFromParent();
+ }
+ }
+
auto *OrigExitingVPV = PhiR->getBackedgeValue();
auto *NewExitingVPV = PhiR->getBackedgeValue();
@@ -6986,7 +6997,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *V;
if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
match(PhiR->getBackedgeValue(),
- m_SelectLike(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
+ m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
// We want code in the middle block to appear to execute on the location of
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ed3d288599aa0..3bd1b689b852d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5708,9 +5708,9 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPValue *BackedgeVal = PhiR->getBackedgeValue();
auto *FindLastSelect = cast<VPSingleDefRecipe>(BackedgeVal);
if (HeaderMask &&
- !match(BackedgeVal, m_SelectLike(m_Specific(HeaderMask),
- m_VPSingleDefRecipe(FindLastSelect),
- m_Specific(PhiR))))
+ !match(BackedgeVal,
+ m_Select(m_Specific(HeaderMask),
+ m_VPSingleDefRecipe(FindLastSelect), m_Specific(PhiR))))
continue;
// Get the find-last expression from the find-last select of the reduction
@@ -6075,9 +6075,9 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// Check if WidenRecipe is the final result of the reduction. If so look
// through selects for predicated reductions.
VPValue *Cond = nullptr;
- VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(vputils::findUserOf(
- WidenRecipe, m_SelectLike(m_VPValue(Cond), m_Specific(WidenRecipe),
- m_Specific(RdxPhi))));
+ VPValue *ExitValue = cast_or_null<VPInstruction>(vputils::findUserOf(
+ WidenRecipe,
+ m_Select(m_VPValue(Cond), m_Specific(WidenRecipe), m_Specific(RdxPhi))));
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue;
assert((!ExitValue || IsLastInChain) &&
@@ -6314,8 +6314,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
- match(ExitValue,
- m_SelectLike(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
+ match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
VPTypeAnalysis &TypeInfo = CostCtx.Types;
SmallVector<VPPartialReductionChain> Chain;
@@ -6450,9 +6449,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
match(R, m_ComputeReductionResult(
m_Specific(Chain.ReductionBinOp))) ||
- match(R,
- m_SelectLike(m_VPValue(), m_Specific(Chain.ReductionBinOp),
- m_Specific(RedPhiR)));
+ match(R, m_Select(m_VPValue(), m_Specific(Chain.ReductionBinOp),
+ m_Specific(RedPhiR)));
};
if (!all_of(Chain.ReductionBinOp->users(), UseIsValid)) {
Chains.clear();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 02c4354b6e3d6..24ff4080d8a2b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -802,7 +802,7 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
// Look through selects inserted for tail folding or predicated reductions.
VPRecipeBase *SelR = vputils::findUserOf(
- BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue()));
+ BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
if (!SelR)
return nullptr;
return vputils::findUserOf<VPInstruction::ComputeReductionResult>(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
index 9c8bb9892c622..add503acb9420 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/select-costs.ll
@@ -69,8 +69,8 @@ exit:
define i32 @select_vpinst_for_tail_folding(i8 %n) {
; CHECK: LV: Checking a loop in 'select_vpinst_for_tail_folding'
-; CHECK: Cost of 6 for VF 2: BLEND vp<{{.+}}> = ir<%red> ir<%red.next>/vp<{{.+}}>
-; CHECK: Cost of 12 for VF 4: BLEND vp<{{.+}}> = ir<%red> ir<%red.next>/vp<{{.+}}>
+; CHECK: Cost of 1 for VF 2: EMIT vp<{{.+}}> = select vp<{{.+}}>, ir<%red.next>, ir<%red>
+; CHECK: Cost of 1 for VF 4: EMIT vp<{{.+}}> = select vp<{{.+}}>, ir<%red.next>, ir<%red>
; CHECK: LV: Selecting VF: 4
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 07116a9d16976..280e3ef8fbb42 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -270,7 +270,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: BLEND vp<[[VP10:%[0-9]+]]> = ir<%and.red> ir<%and.red.next>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%and.red.next>, ir<%and.red>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (and) vp<[[VP10]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
>From ebe09b85e505d72783f568613f2493aaf0da87c6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 11 Jun 2026 21:55:23 +0100
Subject: [PATCH 4/4] !fixup address latest comments.
---
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 3744a520ebfc7..90bd1f67bbd19 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -778,8 +778,9 @@ inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
}
-/// Match a VPBlendRecipe with 2 incoming values ([M0, I0, M1, I1]) as
-/// select(M1, I1, I0). Also matches the normalized form, where M0 is dropped.
+/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
+/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
+/// lowered.
template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
Op0_t MaskOp;
Op1_t TrueOp;
@@ -792,11 +793,9 @@ template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
auto *Blend = dyn_cast<VPBlendRecipe>(Val);
if (!Blend || Blend->getNumIncomingValues() != 2)
return false;
- unsigned TrueIdx = Blend->isNormalized() ? 1 : 0;
- unsigned FalseIdx = 1 - TrueIdx;
- return MaskOp.match(Blend->getMask(TrueIdx)) &&
- TrueOp.match(Blend->getIncomingValue(TrueIdx)) &&
- FalseOp.match(Blend->getIncomingValue(FalseIdx));
+ return MaskOp.match(Blend->getMask(1)) &&
+ TrueOp.match(Blend->getIncomingValue(1)) &&
+ FalseOp.match(Blend->getIncomingValue(0));
}
};
More information about the llvm-commits
mailing list