[llvm] [VPlan] Handle step where sign cannot be determined optimizeFindIVRed. (PR #213450)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 1 07:46:19 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/213450
optimizeFindIVReductions uses the step to determine if min or max is needed. Bail out if the direction of the step cannot be determined via SCEV.
Fixes https://github.com/llvm/llvm-project/issues/213424
>From 9ea67a61d37b0f4e263ae27620bd14cc95610d26 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 1 Aug 2026 13:48:14 +0100
Subject: [PATCH] [VPlan] Handle step where sign cannot be determined
optimizeFindIVRed.
optimizeFindIVReductions uses the step to determine if min or max is
needed. Bail out if the direction of the step cannot be determined via
SCEV.
Fixes https://github.com/llvm/llvm-project/issues/213424
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 37 ++++--
.../Transforms/Vectorize/VPlanTransforms.h | 4 +-
.../find-last-iv-sinkable-expr.ll | 112 ++++++++++++++++++
3 files changed, 140 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fca7f081f8b65..83e23449df5f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4439,6 +4439,22 @@ static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
return ClonedOp;
}
+/// If \p S is an affine AddRec, returns true if its step is known to be
+/// positive and false if it is known to be negative. Returns std::nullopt if
+/// \p S is not an affine AddRec, or if the sign of its step cannot be
+/// determined.
+static std::optional<bool> getStepDirection(const SCEV *S,
+ ScalarEvolution &SE) {
+ const SCEV *Step;
+ if (!match(S, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step))))
+ return std::nullopt;
+ if (SE.isKnownPositive(Step))
+ return true;
+ if (SE.isKnownNegative(Step))
+ return false;
+ return std::nullopt;
+}
+
void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
PredicatedScalarEvolution &PSE,
Loop &L) {
@@ -4502,8 +4518,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
&L);
- const SCEV *Step;
- if (!match(IVSCEV, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step)))) {
+ if (!match(IVSCEV, m_scev_AffineAddRec(m_SCEV(), m_SCEV()))) {
assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
"IVOfExpressionToSink not being an AddRec must imply "
@@ -4511,13 +4526,12 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
continue;
}
- // Determine direction from SCEV step.
- if (!SE.isKnownNonZero(Step))
+ // Determine direction from the step of IVSCEV, if possible.
+ std::optional<bool> StepDirection = getStepDirection(IVSCEV, SE);
+ if (!StepDirection)
continue;
- // Positive step means we need UMax/SMax to find the last IV value, and
- // UMin/SMin otherwise.
- bool UseMax = SE.isKnownPositive(Step);
+ bool UseMax = *StepDirection;
std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
bool UseSigned = SentinelVal && SentinelVal->isSigned();
@@ -4529,16 +4543,15 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
if (IVOfExpressionToSink) {
const SCEV *FindLastExpressionSCEV =
vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L);
- if (match(FindLastExpressionSCEV,
- m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step)))) {
- bool NewUseMax = SE.isKnownPositive(Step);
+ if (std::optional<bool> NewUseMax =
+ getStepDirection(FindLastExpressionSCEV, SE)) {
if (auto NewSentinel =
- CheckSentinel(FindLastExpressionSCEV, NewUseMax)) {
+ CheckSentinel(FindLastExpressionSCEV, *NewUseMax)) {
// The original expression already has a sentinel, so prefer not
// sinking to keep epilogue vectorization possible.
SentinelVal = *NewSentinel;
UseSigned = NewSentinel->isSigned();
- UseMax = NewUseMax;
+ UseMax = *NewUseMax;
IVSCEV = FindLastExpressionSCEV;
IVOfExpressionToSink = nullptr;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 4b59d37150ff4..af8f6de26e0f9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -571,7 +571,9 @@ struct VPlanTransforms {
/// Optimize FindLast reductions selecting IVs (or expressions of IVs) by
/// converting them to FindIV reductions, if their IV range excludes a
/// suitable sentinel value. For expressions of IVs, the expression is sunk
- /// to the middle block.
+ /// to the middle block. The decision is based on SCEV expressions for \p L,
+ /// so this must run before any transform that changes the plan's iteration
+ /// space relative to \p L.
static void optimizeFindIVReductions(VPlan &Plan,
PredicatedScalarEvolution &PSE, Loop &L);
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
index deefaf916c698..43e786c2dd6fe 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
@@ -1069,3 +1069,115 @@ loop:
done:
ret i64 %sel
}
+
+; The multiplier of the sinkable expression %expr is loop-invariant and
+; non-negative, but may be zero, so the step of %expr is not known to be
+; positive. Must not use a min/max reduction of %expr.
+define i32 @findlast_sinkable_expr_step_unknown_sign(ptr %a, i8 %m) {
+; CHECK-LABEL: define i32 @findlast_sinkable_expr_step_unknown_sign(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[M:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL_OP:%.*]] = zext i8 [[M]] to i32
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], [[MUL_OP]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i32 [[TMP4]], -2147483648
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i32 [[TMP5]], i32 -1
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i32 [[TMP7]]
+;
+entry:
+ %mul.op = zext i8 %m to i32
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %load = load i32, ptr %gep, align 4
+ %cmp = icmp eq i32 %load, 42
+ %expr = mul i32 %iv, %mul.op
+ %sel = select i1 %cmp, i32 %expr, i32 %rdx
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exit = icmp eq i32 %iv.next, 8
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i32 %sel
+}
+
+; %step is known to be non-zero (its low bit is set), but its sign is unknown,
+; so %iv2 may be increasing or decreasing. Must not use a min/max reduction.
+; Test for https://github.com/llvm/llvm-project/issues/213424.
+define i32 @findlast_iv_step_nonzero_unknown_sign(ptr %a, i32 %x) {
+; CHECK-LABEL: define i32 @findlast_iv_step_nonzero_unknown_sign(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[STEP:%.*]] = or i32 [[X]], 1
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STEP]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = mul nsw <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i32 [[STEP]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7]] = select i1 [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP6]], <4 x i32> [[VEC_IND]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP8]], <4 x i1> [[TMP7]], i32 -1)
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i32 [[TMP10]]
+;
+entry:
+ %step = or i32 %x, 1
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %iv2 = phi i32 [ 0, %entry ], [ %iv2.next, %loop ]
+ %rdx = phi i32 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i32 %iv
+ %load = load i32, ptr %gep, align 4
+ %cmp = icmp eq i32 %load, 42
+ %sel = select i1 %cmp, i32 %iv2, i32 %rdx
+ %iv2.next = add nsw i32 %iv2, %step
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exit = icmp eq i32 %iv.next, 8
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i32 %sel
+}
More information about the llvm-commits
mailing list