[llvm] [VPlan] Support tailfolded loops in multi-use-reductions (PR #214455)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 04:14:57 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Philipp Rados (prados-oc)
<details>
<summary>Changes</summary>
Previously `handleMultiUseReductions()` would bail out for tailfolded-loops, since the backedge value is no longer the reduction intrinsic but the predicated select based on the header-mask.
This patch adds pattern matching to handle tailfolded loops accordingly.
I ran the LLVM testsuite (for RISCV with -march=rv64gcv) and it only triggers in the corresponding unit-tests in `SingleSource/UnitTests/Vectorizer/`. I guess that's because there are still some other limitations when handling more generic multi-use reduction cases (e.g. differing types in CanonicalIV vs. WideIV).
---
Patch is 27.19 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214455.diff
4 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+8-5)
- (modified) llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp (+49-17)
- (modified) llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll (+156)
- (added) llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll (+128)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a0abc151ace19..ad636d80e0e61 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1243,7 +1243,7 @@ class LoopVectorizationCostModel {
/// Returns true if the predicated reduction select should be used to set the
/// incoming value for the reduction phi.
- bool usePredicatedReductionSelect(RecurKind RecurrenceKind) const {
+ bool usePredicatedReductionSelect(const RecurrenceDescriptor &RdxDesc) const {
// Force to use predicated reduction select since the EVL of the
// second-to-last iteration might not be VF*UF.
if (foldTailWithEVL())
@@ -1254,10 +1254,13 @@ class LoopVectorizationCostModel {
if (maskPartialAliasing())
return true;
- // Note: For FindLast recurrences we prefer a predicated select to simplify
- // matching in handleFindLastReductions(), rather than handle multiple
+ // Note: For FindLast recurrences and multi-use reductions we prefer a
+ // predicated select to simplify matching in handleFindLastReductions() and
+ // handleMultiUseReductions() respectively, rather than handle multiple
// cases.
- if (RecurrenceDescriptor::isFindLastRecurrenceKind(RecurrenceKind))
+ if (RecurrenceDescriptor::isFindLastRecurrenceKind(
+ RdxDesc.getRecurrenceKind()) ||
+ RdxDesc.hasUsesOutsideReductionChain())
return true;
return PreferPredicatedReductionSelect ||
@@ -6937,7 +6940,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
// Remove the predicated select if the target doesn't want it.
VPValue *V;
- if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
+ if (!CM.usePredicatedReductionSelect(RdxDesc) &&
match(PhiR->getBackedgeValue(),
m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
PhiR->setBackedgeValue(V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 69d02c63a3def..10898ce122095 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1936,8 +1936,16 @@ static bool handleFirstArgMinOrMax(
if (Ty != WideIV->getScalarType())
return false;
- auto *FindIVSelectR = cast<VPSingleDefRecipe>(
- FindLastIVPhiR->getBackedgeValue()->getDefiningRecipe());
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
+ auto *BackedgeVal = FindLastIVPhiR->getBackedgeValue();
+ auto *FindIVSelectVal = BackedgeVal;
+ if (HeaderMask && !match(BackedgeVal, m_Select(m_Specific(HeaderMask),
+ m_VPValue(FindIVSelectVal),
+ m_Specific(FindLastIVPhiR))))
+ return false;
+ auto *FindIVSelectR =
+ cast<VPSingleDefRecipe>(FindIVSelectVal->getDefiningRecipe());
+
assert(
match(FindIVSelectR, m_Select(m_VPValue(), m_VPValue(), m_VPValue())) &&
"backedge value must be a select");
@@ -2067,32 +2075,51 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
// MinOrMaxPhiR has users outside the reduction cycle in the loop. Check if
// the only other user is a FindLastIV reduction. MinOrMaxPhiR must have
- // exactly 2 users:
+ // exactly 2 users if not tailfolded:
// 1) the min/max operation of the reduction cycle, and
// 2) the compare of a FindLastIV reduction cycle. This compare must match
// the min/max operation - comparing MinOrMaxPhiR with the operand of the
// min/max operation, and be used only by the select of the FindLastIV
// reduction cycle.
+ // There is an additional user if the loop was tailfolded:
+ // 3) the select operation of the vector.latch block. This select uses the
+ // the original MinOrMaxPhiR if the mask is zero.
RecurKind RdxKind = MinOrMaxPhiR->getRecurrenceKind();
assert(
RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind) &&
"only min/max recurrences support users outside the reduction chain");
- auto *MinOrMaxOp =
+ auto *MinOrMaxBackedgeR =
dyn_cast<VPRecipeWithIRFlags>(MinOrMaxPhiR->getBackedgeValue());
- if (!MinOrMaxOp)
+ if (!MinOrMaxBackedgeR)
return false;
- // Check that MinOrMaxOp is a VPWidenIntrinsicRecipe or VPReplicateRecipe
- // with an intrinsic that matches the reduction kind.
+ // If the loop is tailfolded then the backedge won't be the
+ // reduction-intrinsic but the select in the vector.latch block that wraps
+ // the reduction-intrinsic.
+ auto *MinOrMaxOp = MinOrMaxBackedgeR;
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
+ if (VPValue *MinOrMaxTailfold;
+ HeaderMask &&
+ match(MinOrMaxBackedgeR,
+ m_SelectLike(m_Specific(HeaderMask), m_VPValue(MinOrMaxTailfold),
+ m_Specific(MinOrMaxPhiR)))) {
+ MinOrMaxOp =
+ dyn_cast<VPRecipeWithIRFlags>(MinOrMaxTailfold->getDefiningRecipe());
+ if (!MinOrMaxOp)
+ return false;
+ }
+
+ // Check that MinOrMaxOp is a VPWidenIntrinsicRecipe or
+ // VPReplicateRecipe with an intrinsic that matches the reduction kind.
Intrinsic::ID ExpectedIntrinsicID = getMinMaxReductionIntrinsicOp(RdxKind);
if (!match(MinOrMaxOp, m_Intrinsic(ExpectedIntrinsicID)))
return false;
- // MinOrMaxOp must have 2 users: 1) MinOrMaxPhiR and 2)
+ // MinOrMaxBackedgeR must have 2 users: 1) MinOrMaxPhiR and 2)
// ComputeReductionResult.
- assert(MinOrMaxOp->getNumUsers() == 2 &&
- "MinOrMaxOp must have exactly 2 users");
+ assert(MinOrMaxBackedgeR->getNumUsers() == 2 &&
+ "MinOrMaxBackedgeR must have exactly 2 users");
VPValue *MinOrMaxOpValue = MinOrMaxOp->getOperand(0);
if (MinOrMaxOpValue == MinOrMaxPhiR)
MinOrMaxOpValue = MinOrMaxOp->getOperand(1);
@@ -2109,22 +2136,27 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan,
if (MinOrMaxOpValue != CmpOpB)
Pred = CmpInst::getSwappedPredicate(Pred);
- // MinOrMaxPhiR must have exactly 2 users:
+ // MinOrMaxPhiR must have exactly 2 users if not tailfolded:
// * MinOrMaxOp,
// * Cmp (that's part of a FindLastIV chain).
- if (MinOrMaxPhiR->getNumUsers() != 2)
+ // Also an additional third user if it is tailfolded:
+ // * Predicated HEADER-MASK select in the vector.latch block.
+ if ((!HeaderMask && MinOrMaxPhiR->getNumUsers() != 2) ||
+ (HeaderMask && MinOrMaxPhiR->getNumUsers() != 3))
return false;
VPInstruction *MinOrMaxResult =
- findUserOf<VPInstruction::ComputeReductionResult>(MinOrMaxOp);
- assert(is_contained(MinOrMaxPhiR->users(), MinOrMaxOp) &&
- "one user must be MinOrMaxOp");
- assert(MinOrMaxResult && "MinOrMaxResult must be a user of MinOrMaxOp");
+ findUserOf<VPInstruction::ComputeReductionResult>(MinOrMaxBackedgeR);
+ assert(is_contained(MinOrMaxPhiR->users(), MinOrMaxBackedgeR) &&
+ "one user must be MinOrMaxBackedgeR");
+ assert(MinOrMaxResult &&
+ "MinOrMaxResult must be a user of MinOrMaxBackedgeR");
// Cmp must be used by the select of a FindLastIV chain.
VPValue *Sel = dyn_cast<VPSingleDefRecipe>(Cmp->getSingleUser());
VPValue *IVOp, *FindIV;
- if (!Sel || Sel->getNumUsers() != 2 ||
+ if (!Sel || (HeaderMask && Sel->getNumUsers() != 1) ||
+ (!HeaderMask && Sel->getNumUsers() != 2) ||
!match(Sel,
m_Select(m_Specific(Cmp), m_VPValue(IVOp), m_VPValue(FindIV))))
return false;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
index 1ca4b6fdb2283..8c5e4a93c0898 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
@@ -1678,6 +1678,162 @@ for.end:
}
+; Test for multiuse-reductions that are tailfolded.
+define dso_local signext i32 @argmin(ptr %arr, i32 %N) {
+; IF-EVL-LABEL: @argmin(
+; IF-EVL-NEXT: entry:
+; IF-EVL-NEXT: [[CONV:%.*]] = sext i32 [[N:%.*]] to i64
+; IF-EVL-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[N]], 0
+; IF-EVL-NEXT: br i1 [[CMP15]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IF-EVL: for.body.preheader:
+; IF-EVL-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARR:%.*]], align 8
+; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]]
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP0]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP1:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]]
+; IF-EVL: vector.body:
+; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ poison, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[CONV]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[INDEX]]
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
+; IF-EVL-NEXT: [[TMP5:%.*]] = icmp slt <vscale x 2 x i64> [[VP_OP_LOAD]], [[VEC_PHI]]
+; IF-EVL-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.smin.nxv2i64(<vscale x 2 x i64> [[VP_OP_LOAD]], <vscale x 2 x i64> [[VEC_PHI]])
+; IF-EVL-NEXT: [[TMP7]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[TMP6]], <vscale x 2 x i64> [[VEC_PHI]], i32 [[TMP2]])
+; IF-EVL-NEXT: [[TMP8]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP5]], <vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x i64> [[VEC_PHI1]], i32 [[TMP2]])
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i64 [[TMP3]], [[INDEX]]
+; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]]
+; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT3]]
+; IF-EVL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; IF-EVL-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; IF-EVL: middle.block:
+; IF-EVL-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> [[TMP7]])
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT4]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP11:%.*]] = icmp eq <vscale x 2 x i64> [[TMP7]], [[BROADCAST_SPLAT5]]
+; IF-EVL-NEXT: [[TMP12:%.*]] = select <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> splat (i64 -1)
+; IF-EVL-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> [[TMP12]])
+; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP10]], [[TMP0]]
+; IF-EVL-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i64 0, i64 [[TMP13]]
+; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
+; IF-EVL: for.cond.cleanup.loopexit:
+; IF-EVL-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP]]
+; IF-EVL: for.cond.cleanup:
+; IF-EVL-NEXT: [[MINLOC_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP16]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; IF-EVL-NEXT: ret i32 [[MINLOC_0_LCSSA]]
+;
+; NO-VP-LABEL: @argmin(
+; NO-VP-NEXT: entry:
+; NO-VP-NEXT: [[CONV:%.*]] = sext i32 [[N:%.*]] to i64
+; NO-VP-NEXT: [[CMP15:%.*]] = icmp sgt i32 [[N]], 0
+; NO-VP-NEXT: br i1 [[CMP15]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; NO-VP: for.body.preheader:
+; NO-VP-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARR:%.*]], align 8
+; NO-VP-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CONV]], [[TMP2]]
+; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP: vector.ph:
+; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
+; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[CONV]], [[TMP3]]
+; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[CONV]], [[N_MOD_VF]]
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP0]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-VP-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]]
+; NO-VP: vector.body:
+; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ poison, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 8
+; NO-VP-NEXT: [[TMP6:%.*]] = icmp slt <vscale x 2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
+; NO-VP-NEXT: [[TMP7]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x i64> [[VEC_PHI3]]
+; NO-VP-NEXT: [[TMP8]] = call <vscale x 2 x i64> @llvm.smin.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[VEC_PHI]])
+; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; NO-VP-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; NO-VP: middle.block:
+; NO-VP-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.smin.nxv2i64(<vscale x 2 x i64> [[TMP8]])
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT4]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-VP-NEXT: [[TMP11:%.*]] = icmp eq <vscale x 2 x i64> [[TMP8]], [[BROADCAST_SPLAT5]]
+; NO-VP-NEXT: [[TMP12:%.*]] = select <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> splat (i64 -1)
+; NO-VP-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.umin.nxv2i64(<vscale x 2 x i64> [[TMP12]])
+; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP10]], [[TMP0]]
+; NO-VP-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i64 0, i64 [[TMP13]]
+; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[CONV]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; NO-VP: scalar.ph:
+; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[FOR_BODY_PREHEADER]] ]
+; NO-VP-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i64 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; NO-VP-NEXT: br label [[FOR_BODY:%.*]]
+; NO-VP: for.cond.cleanup.loopexit:
+; NO-VP-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT:%.*]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; NO-VP-NEXT: [[TMP16:%.*]] = trunc i64 [[SPEC_SELECT_LCSSA]] to i32
+; NO-VP-NEXT: br label [[FOR_COND_CLEANUP]]
+; NO-VP: for.cond.cleanup:
+; NO-VP-NEXT: [[MINLOC_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP16]], [[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; NO-VP-NEXT: ret i32 [[MINLOC_0_LCSSA]]
+; NO-VP: for.body:
+; NO-VP-NEXT: [[I_018:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; NO-VP-NEXT: [[MIN_017:%.*]] = phi i64 [ [[SPEC_SELECT14:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; NO-VP-NEXT: [[MINLOC_016:%.*]] = phi i64 [ [[SPEC_SELECT]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX6]], [[SCALAR_PH]] ]
+; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARR]], i64 [[I_018]]
+; NO-VP-NEXT: [[TMP17:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
+; NO-VP-NEXT: [[CMP3:%.*]] = icmp slt i64 [[TMP17]], [[MIN_017]]
+; NO-VP-NEXT: [[SPEC_SELECT]] = select i1 [[CMP3]], i64 [[I_018]], i64 [[MINLOC_016]]
+; NO-VP-NEXT: [[SPEC_SELECT14]] = tail call i64 @llvm.smin.i64(i64 [[TMP17]], i64 [[MIN_017]])
+; NO-VP-NEXT: [[INC]] = add nuw nsw i64 [[I_018]], 1
+; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[CONV]]
+; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+;
+entry:
+ %conv = sext i32 %N to i64
+ %cmp15 = icmp sgt i32 %N, 0
+ br i1 %cmp15, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %0 = load i64, ptr %arr, align 8
+ br label %for.body
+
+for.cond.cleanup.loopexit:
+ %spec.select.lcssa = phi i64 [ %spec.select, %for.body ]
+ %1 = trunc i64 %spec.select.lcssa to i32
+ br label %for.cond.cleanup
+
+for.cond.cleanup:
+ %minloc.0.lcssa = phi i32 [ 0, %entry ], [ %1, %for.cond.cleanup.loopexit ]
+ ret i32 %minloc.0.lcssa
+
+for.body:
+ %i.018 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+ %min.017 = phi i64 [ %spec.select14, %for.body ], [ %0, %for.body.preheader ]
+ %minloc.016 = phi i64 [ %spec.select, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx2 = getelementptr inbounds nuw [8 x i8], ptr %arr, i64 %i.018
+ %2 = load i64, ptr %arrayidx2, align 8
+ %cmp3 = icmp slt i64 %2, %min.017
+ %spec.select = select i1 %cmp3, i64 %i.018, i64 %minloc.016
+ %spec.select14 = tail call i64 @llvm.smin.i64(i64 %2, i64 %min.017)
+ %inc = add nuw nsw i64 %i.018, 1
+ %exitcond.not = icmp eq i64 %inc, %conv
+ br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !0
+}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll
new file mode 100644
index 0000000000000..a58f192941cc1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding-multiuse-reductions.ll
@@ -0,0 +1,128 @@
+; NOTE: Assertions have been autog...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/214455
More information about the llvm-commits
mailing list