[llvm] [VPlan] Expand sequential/regular UMin SCEVs in VPSCEVExpander. (PR #209786)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 11:35:31 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/209786
>From c43f4fc7fbcc573cfaf64e820f013b4883c1bdcc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 14 Jul 2026 11:02:21 +0100
Subject: [PATCH 1/3] [VPlan] Expand sequential/regular UMin SCEVs in
VPSCEVExpander.
Add support for expanding SequentialUMinExpr SCEV expressions in
VPSCEVExpander.
For regular UMin expressions, the expansion unconditionally expands &
executes all operands, while the semantics of sequential UMin only
require the first operand to be evaluated unconditionally.
For sequential UMin expressions, we need to make sure potentially
UB/poison generating operands must be accounted for. Matching IR SCEV
expander, make sure that divisors of UDiv are poison-free and non-zero
inside sequential UMin. Similarly, freeze all operands other than the
first, to avoid poison from propagating.
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 51 ++++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 ++
.../trip-count-expansion-may-introduce-ub.ll | 2 +-
3 files changed, 42 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e460f63953ac4..75003e43bc7b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -945,9 +945,23 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
VPValue *LHS = tryToExpand(UDiv->getLHS());
if (!LHS)
return nullptr;
- VPValue *RHS = tryToExpand(UDiv->getRHS());
+ const SCEV *RHSExpr = UDiv->getRHS();
+ VPValue *RHS = tryToExpand(RHSExpr);
if (!RHS)
return nullptr;
+ if (SafeUDivMode) {
+ // Make sure the UDiv's divisor is guaranteed to not be zero/poison, to
+ // avoid UB.
+ Type *RHSTy = RHSExpr->getType();
+ bool GuaranteedNotPoison =
+ ScalarEvolution::isGuaranteedNotToBePoison(RHSExpr);
+ if (!GuaranteedNotPoison)
+ RHS = Builder.createScalarFreeze(RHS, RHSTy, DL);
+ if (!SE.isKnownNonZero(RHSExpr) || !GuaranteedNotPoison)
+ RHS = Builder.createScalarIntrinsic(
+ Intrinsic::umax, {RHS, Builder.getPlan().getConstantInt(RHSTy, 1)},
+ RHSTy, DL);
+ }
return Builder.createNaryOp(Instruction::UDiv, {LHS, RHS},
VPIRFlags::getDefaultFlags(Instruction::UDiv),
DL);
@@ -998,8 +1012,9 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
case scUMaxExpr:
case scSMaxExpr:
case scUMinExpr:
- case scSMinExpr: {
- auto *MinMax = cast<SCEVMinMaxExpr>(S);
+ case scSMinExpr:
+ case scSequentialUMinExpr: {
+ auto *MinMax = cast<SCEVNAryExpr>(S);
Intrinsic::ID IntrinsicID;
switch (S->getSCEVType()) {
case scUMaxExpr:
@@ -1009,6 +1024,7 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
IntrinsicID = Intrinsic::smax;
break;
case scUMinExpr:
+ case scSequentialUMinExpr:
IntrinsicID = Intrinsic::umin;
break;
case scSMinExpr:
@@ -1018,19 +1034,26 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
llvm_unreachable("Unexpected min/max SCEV type");
}
// Chain operands in reverse order matching SCEVExpander's expansion of
- // min/max expressions.
- SmallVector<VPValue *, 2> Ops;
- for (const SCEVUse &Op : reverse(MinMax->operands())) {
- VPValue *OpV = tryToExpand(Op);
- if (!OpV)
+ // min/max expressions. In SafeUDivMode freeze expansion results of operands
+ // other than the first for sequential UMins, to avoid short-circuiting
+ // divide-by-0/poison.
+ bool IsSequential = S->getSCEVType() == scSequentialUMinExpr;
+ Type *ResultTy = MinMax->getType();
+ bool PrevSafeMode = SafeUDivMode;
+ VPValue *Result = nullptr;
+ for (const auto &[I, SCEVOp] : enumerate(reverse(MinMax->operands()))) {
+ bool MayShortCircuit = IsSequential && I != MinMax->getNumOperands() - 1;
+ SafeUDivMode = MayShortCircuit || PrevSafeMode;
+ VPValue *Op = tryToExpand(SCEVOp);
+ SafeUDivMode = PrevSafeMode;
+ if (!Op)
return nullptr;
- Ops.push_back(OpV);
+ if (MayShortCircuit)
+ Op = Builder.createScalarFreeze(Op, ResultTy, DL);
+ Result = Result ? Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
+ ResultTy, DL)
+ : Op;
}
- Type *ResultTy = MinMax->getType();
- VPValue *Result = Ops.front();
- for (VPValue *Op : drop_begin(Ops))
- Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
- ResultTy, DL);
return Result;
}
default:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..60a9a8013de66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -252,6 +252,10 @@ class VPSCEVExpander {
ScalarEvolution &SE;
DebugLoc DL;
+ /// When true, nested SCEVUDivExprs are expanded so that they cannot divide by
+ /// zero, matching SCEVExpander's SafeUDivMode.
+ bool SafeUDivMode = false;
+
/// Try to find a loop-invariant IR value in the plan's entry block whose
/// SCEV matches \p S. Returns the corresponding live-in VPValue, or nullptr
/// if none is found.
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 662cf84b676bc..8ae7439164123 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -916,7 +916,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP12]]
-; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[N]], [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 42, [[TMP1]]
; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[SMAX1]]
>From c7903221ac499f2e7aff21f1351014edd4c7d445 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 14 Aug 2026 12:44:50 +0100
Subject: [PATCH 2/3] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 29 +++++++++++---------
1 file changed, 16 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 75003e43bc7b2..bba5afe981ff9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -952,15 +952,15 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
if (SafeUDivMode) {
// Make sure the UDiv's divisor is guaranteed to not be zero/poison, to
// avoid UB.
- Type *RHSTy = RHSExpr->getType();
+ Type *Ty = UDiv->getType();
bool GuaranteedNotPoison =
ScalarEvolution::isGuaranteedNotToBePoison(RHSExpr);
if (!GuaranteedNotPoison)
- RHS = Builder.createScalarFreeze(RHS, RHSTy, DL);
+ RHS = Builder.createScalarFreeze(RHS, Ty, DL);
if (!SE.isKnownNonZero(RHSExpr) || !GuaranteedNotPoison)
RHS = Builder.createScalarIntrinsic(
- Intrinsic::umax, {RHS, Builder.getPlan().getConstantInt(RHSTy, 1)},
- RHSTy, DL);
+ Intrinsic::umax, {RHS, Builder.getPlan().getConstantInt(Ty, 1)}, Ty,
+ DL);
}
return Builder.createNaryOp(Instruction::UDiv, {LHS, RHS},
VPIRFlags::getDefaultFlags(Instruction::UDiv),
@@ -1040,20 +1040,23 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
bool IsSequential = S->getSCEVType() == scSequentialUMinExpr;
Type *ResultTy = MinMax->getType();
bool PrevSafeMode = SafeUDivMode;
- VPValue *Result = nullptr;
- for (const auto &[I, SCEVOp] : enumerate(reverse(MinMax->operands()))) {
- bool MayShortCircuit = IsSequential && I != MinMax->getNumOperands() - 1;
+ SmallVector<VPValue *, 2> Ops;
+ for (const SCEV *SCEVOp : reverse(MinMax->operands())) {
+ bool MayShortCircuit =
+ IsSequential && Ops.size() != MinMax->getNumOperands() - 1;
SafeUDivMode = MayShortCircuit || PrevSafeMode;
- VPValue *Op = tryToExpand(SCEVOp);
+ VPValue *OpV = tryToExpand(SCEVOp);
SafeUDivMode = PrevSafeMode;
- if (!Op)
+ if (!OpV)
return nullptr;
if (MayShortCircuit)
- Op = Builder.createScalarFreeze(Op, ResultTy, DL);
- Result = Result ? Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
- ResultTy, DL)
- : Op;
+ OpV = Builder.createScalarFreeze(OpV, ResultTy, DL);
+ Ops.push_back(OpV);
}
+ VPValue *Result = Ops.front();
+ for (VPValue *Op : drop_begin(Ops))
+ Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
+ ResultTy, DL);
return Result;
}
default:
>From 0a6f8b3bd4bc141866d5d0064091ceddec34c539 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 21 Aug 2026 20:58:45 +0100
Subject: [PATCH 3/3] !fixup update after rebase
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 ++--
llvm/test/Transforms/LoopVectorize/X86/cost-model.ll | 3 ++-
.../LoopVectorize/trip-count-expansion-may-introduce-ub.ll | 2 +-
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index bba5afe981ff9..e6b267493d987 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -956,7 +956,7 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
bool GuaranteedNotPoison =
ScalarEvolution::isGuaranteedNotToBePoison(RHSExpr);
if (!GuaranteedNotPoison)
- RHS = Builder.createScalarFreeze(RHS, Ty, DL);
+ RHS = Builder.createScalarFreeze(RHS, DL);
if (!SE.isKnownNonZero(RHSExpr) || !GuaranteedNotPoison)
RHS = Builder.createScalarIntrinsic(
Intrinsic::umax, {RHS, Builder.getPlan().getConstantInt(Ty, 1)}, Ty,
@@ -1050,7 +1050,7 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
if (!OpV)
return nullptr;
if (MayShortCircuit)
- OpV = Builder.createScalarFreeze(OpV, ResultTy, DL);
+ OpV = Builder.createScalarFreeze(OpV, DL);
Ops.push_back(OpV);
}
VPValue *Result = Ops.front();
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 286bf2ebf47f5..2c740670be578 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -449,7 +449,8 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-LABEL: define void @multi_exit(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], i64 [[A:%.*]], i64 [[B:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1)
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[B]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[TMP10]], -1
; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]]
; CHECK-NEXT: [[UMIN10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[UMIN10]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
index 8ae7439164123..662cf84b676bc 100644
--- a/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+++ b/llvm/test/Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
@@ -916,7 +916,7 @@ define i64 @multi_exit_4_exit_count_with_urem_by_value_in_latch(ptr %dst, i64 %N
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 42, [[TMP12]]
-; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[N]], [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 42, [[TMP1]]
; CHECK-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP2]], i64 0)
; CHECK-NEXT: [[TMP10:%.*]] = freeze i64 [[SMAX1]]
More information about the llvm-commits
mailing list