[llvm] 80cc28b - [VPlan] Normalize selects in optimizeFindIVReductions. (#191517)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 14:47:43 PDT 2026
Author: Florian Hahn
Date: 2026-04-10T22:47:38+01:00
New Revision: 80cc28b6689876f740fcbb956ed6926218b33719
URL: https://github.com/llvm/llvm-project/commit/80cc28b6689876f740fcbb956ed6926218b33719
DIFF: https://github.com/llvm/llvm-project/commit/80cc28b6689876f740fcbb956ed6926218b33719.diff
LOG: [VPlan] Normalize selects in optimizeFindIVReductions. (#191517)
Follow-up to https://github.com/llvm/llvm-project/pull/191166.
Unify code to create new selects in optmizeFindIVReductions to simplify
the code paths and make it easier to include the header mask correctly
if needed.
If no new select needs to be created, just use the original backedge
value.
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b8fc04f17f0b6..d46f05f9ead57 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5878,32 +5878,30 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
continue;
}
- // When tail folding, mask the condition with the header mask to prevent
- // propagating poison from inactive lanes in the last vector iteration.
- VPValue *SelectCond = Cond;
- if (HeaderMask) {
- VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
- SelectCond = LoopBuilder.createLogicalAnd(HeaderMask, Cond);
- }
-
VPInstruction *RdxResult = cast<VPInstruction>(vputils::findRecipe(
BackedgeVal,
match_fn(m_VPInstruction<VPInstruction::ComputeReductionResult>())));
- // If IV is an expression to sink, create a new select that tracks the
- // underlying IV directly and update the backedge value.
- if (IVOfExpressionToSink) {
- auto *OrigFindLastSelectR = FindLastSelect->getDefiningRecipe();
- VPBuilder LoopBuilder(OrigFindLastSelectR);
- DebugLoc DL = OrigFindLastSelectR->getDebugLoc();
- if (OrigFindLastSelectR->getOperand(1) == PhiR) {
- FindLastSelect = LoopBuilder.createSelect(SelectCond, PhiR,
- IVOfExpressionToSink, DL);
- } else {
- assert(OrigFindLastSelectR->getOperand(2) == PhiR &&
- "PhiR expected as operand 1 or 2");
- FindLastSelect = LoopBuilder.createSelect(
- SelectCond, IVOfExpressionToSink, PhiR, DL);
+ VPValue *NewFindLastSelect = BackedgeVal;
+ VPValue *SelectCond = Cond;
+ if (!SentinelVal || IVOfExpressionToSink) {
+ // When we need to create a new select, normalize the condition so that
+ // PhiR is the last operand and include the header mask if needed.
+ DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
+ VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
+ if (FindLastSelect->getDefiningRecipe()->getOperand(1) == PhiR)
+ SelectCond = LoopBuilder.createNot(SelectCond);
+
+ // When tail folding, mask the condition with the header mask to prevent
+ // propagating poison from inactive lanes in the last vector iteration.
+ if (HeaderMask)
+ SelectCond = LoopBuilder.createLogicalAnd(HeaderMask, SelectCond);
+
+ if (SelectCond != Cond || IVOfExpressionToSink) {
+ NewFindLastSelect = LoopBuilder.createSelect(
+ SelectCond,
+ IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
+ PhiR, DL);
}
}
@@ -5915,8 +5913,9 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
FastMathFlags());
DebugLoc ExitDL = RdxResult->getDebugLoc();
VPBuilder MiddleBuilder(RdxResult);
- VPValue *ReducedIV = MiddleBuilder.createNaryOp(
- VPInstruction::ComputeReductionResult, FindLastSelect, Flags, ExitDL);
+ VPValue *ReducedIV =
+ MiddleBuilder.createNaryOp(VPInstruction::ComputeReductionResult,
+ NewFindLastSelect, Flags, ExitDL);
// If IVOfExpressionToSink is an expression to sink, sink it now.
VPValue *VectorRegionExitingVal = ReducedIV;
@@ -5946,14 +5945,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
AnyOfPhi->insertAfter(PhiR);
VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
- VPValue *AnyOfCond = Cond;
- if (FindLastSelect->getOperand(1) == PhiR)
- AnyOfCond = LoopBuilder.createNot(Cond);
- // When tail folding, mask the condition with the header mask to prevent
- // propagating poison from inactive lanes in the last vector iteration.
- if (HeaderMask)
- AnyOfCond = LoopBuilder.createLogicalAnd(HeaderMask, AnyOfCond);
- VPValue *OrVal = LoopBuilder.createOr(AnyOfPhi, AnyOfCond);
+ VPValue *OrVal = LoopBuilder.createOr(AnyOfPhi, SelectCond);
AnyOfPhi->setOperand(1, OrVal);
VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
@@ -5974,7 +5966,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
auto *NewPhiR = new VPReductionPHIRecipe(
cast<PHINode>(PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
- *FindLastSelect, RdxUnordered{1}, {},
+ *NewFindLastSelect, RdxUnordered{1}, {},
PhiR->hasUsesOutsideReductionChain());
NewPhiR->insertBefore(PhiR);
PhiR->replaceAllUsesWith(NewPhiR);
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
index e5838bf9d97af..0554be0b0cc70 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
@@ -516,7 +516,7 @@ define i64 @findlast_non_canonical_iv_with_expr(ptr %a, i64 %n) {
; TF: [[VECTOR_BODY]]:
; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8:.*]] ]
; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 10, i64 12, i64 14, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
+; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
; TF-NEXT: [[TMP3:%.*]] = mul i64 [[INDEX]], 2
; TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 10, [[TMP3]]
; TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
@@ -565,13 +565,14 @@ define i64 @findlast_non_canonical_iv_with_expr(ptr %a, i64 %n) {
; TF-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP27]], %[[PRED_LOAD_IF7]] ]
; TF-NEXT: [[TMP29:%.*]] = icmp eq <4 x i64> [[TMP28]], splat (i64 42)
; TF-NEXT: [[TMP30:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 100)
-; TF-NEXT: [[TMP31]] = select <4 x i1> [[TMP29]], <4 x i64> [[TMP30]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP29]], <4 x i64> [[TMP30]], <4 x i64> [[VEC_PHI]]
+; TF-NEXT: [[TMP36]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP31]], <4 x i64> [[VEC_PHI]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)
; TF-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TF-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP33:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP31]])
+; TF-NEXT: [[TMP33:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP36]])
; TF-NEXT: [[TMP34:%.*]] = icmp ne i64 [[TMP33]], 0
; TF-NEXT: [[TMP35:%.*]] = select i1 [[TMP34]], i64 [[TMP33]], i64 -1
; TF-NEXT: br label %[[DONE:.*]]
@@ -1198,8 +1199,8 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; TF-NEXT: br label %[[VECTOR_BODY:.*]]
; TF: [[VECTOR_BODY]]:
; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; TF-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; TF-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; TF-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
@@ -1216,14 +1217,12 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; TF-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP6]], i64 -3
; TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP7]], <4 x i1> [[REVERSE]], <4 x i64> poison)
; TF-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x i64> [[WIDE_MASKED_LOAD8]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; TF-NEXT: [[TMP16:%.*]] = icmp sgt <4 x i64> [[REVERSE6]], [[REVERSE9]]
+; TF-NEXT: [[TMP16:%.*]] = icmp sle <4 x i64> [[REVERSE6]], [[REVERSE9]]
; TF-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP16]], <4 x i1> zeroinitializer
; TF-NEXT: [[TMP9]] = select <4 x i1> [[TMP8]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[TMP17:%.*]] = xor <4 x i1> [[TMP16]], splat (i1 true)
-; TF-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP17]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI3]], [[TMP18]]
+; TF-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI3]], [[TMP8]]
; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_PHI]], splat (i64 -4)
+; TF-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
; TF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; TF: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index b8ef3b0e464e9..a8ac167c4f6f3 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -330,11 +330,15 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP76]], <4 x i16> [[TMP77]])
-; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP70]])
-; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP71]])
+; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
+; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
+; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
; IC4VF4-NEXT: [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
; IC4VF4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
; IC4VF4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
@@ -592,11 +596,15 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
; IC4VF4-NEXT: [[TMP77:%.*]] = select <4 x i1> [[TMP101]], <4 x i16> <i16 7, i16 6, i16 5, i16 4>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[TMP70:%.*]] = select <4 x i1> [[TMP102]], <4 x i16> <i16 3, i16 2, i16 1, i16 0>, <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: [[TMP71:%.*]] = select <4 x i1> [[TMP103]], <4 x i16> <i16 -1, i16 -2, i16 -3, i16 -4>, <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP82:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP76]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP83:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP77]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP88:%.*]] = select <4 x i1> splat (i1 true), <4 x i16> [[TMP70]], <4 x i16> splat (i16 32767)
+; IC4VF4-NEXT: [[TMP89:%.*]] = select <4 x i1> zeroinitializer, <4 x i16> [[TMP71]], <4 x i16> splat (i16 32767)
; IC4VF4-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; IC4VF4: [[MIDDLE_BLOCK]]:
-; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP76]], <4 x i16> [[TMP77]])
-; IC4VF4-NEXT: [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP70]])
-; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP71]])
+; IC4VF4-NEXT: [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP82]], <4 x i16> [[TMP83]])
+; IC4VF4-NEXT: [[RDX_MINMAX31:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP88]])
+; IC4VF4-NEXT: [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX31]], <4 x i16> [[TMP89]])
; IC4VF4-NEXT: [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
; IC4VF4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
; IC4VF4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
More information about the llvm-commits
mailing list