[llvm] fc9f8ec - [VPlan] Expand sequential/regular UMin SCEVs in VPSCEVExpander. (#209786)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 12:18:24 PDT 2026
Author: Florian Hahn
Date: 2026-08-22T19:18:19Z
New Revision: fc9f8ec195b9dd29faaf73cb3dfe058df3ddd63d
URL: https://github.com/llvm/llvm-project/commit/fc9f8ec195b9dd29faaf73cb3dfe058df3ddd63d
DIFF: https://github.com/llvm/llvm-project/commit/fc9f8ec195b9dd29faaf73cb3dfe058df3ddd63d.diff
LOG: [VPlan] Expand sequential/regular UMin SCEVs in VPSCEVExpander. (#209786)
Add support for expanding SequentialUMinExpr SCEV expressions in
VPSCEVExpander.
For regular UMin expressions, the expansion unconditionally expands &
executes all operands, while the semantics of sequential UMin only
require the first operand to be evaluated unconditionally.
For sequential UMin expressions, we need to make sure potentially
UB/poison generating operands must be accounted for. Matching IR SCEV
expander, make sure that divisors of UDiv are poison-free and non-zero
inside sequential UMin. Similarly, freeze all operands other than the
first, to avoid poison from propagating.
PR: https://github.com/llvm/llvm-project/pull/209786
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.h
llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e460f63953ac4..e6b267493d987 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -945,9 +945,23 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
VPValue *LHS = tryToExpand(UDiv->getLHS());
if (!LHS)
return nullptr;
- VPValue *RHS = tryToExpand(UDiv->getRHS());
+ const SCEV *RHSExpr = UDiv->getRHS();
+ VPValue *RHS = tryToExpand(RHSExpr);
if (!RHS)
return nullptr;
+ if (SafeUDivMode) {
+ // Make sure the UDiv's divisor is guaranteed to not be zero/poison, to
+ // avoid UB.
+ Type *Ty = UDiv->getType();
+ bool GuaranteedNotPoison =
+ ScalarEvolution::isGuaranteedNotToBePoison(RHSExpr);
+ if (!GuaranteedNotPoison)
+ RHS = Builder.createScalarFreeze(RHS, DL);
+ if (!SE.isKnownNonZero(RHSExpr) || !GuaranteedNotPoison)
+ RHS = Builder.createScalarIntrinsic(
+ Intrinsic::umax, {RHS, Builder.getPlan().getConstantInt(Ty, 1)}, Ty,
+ DL);
+ }
return Builder.createNaryOp(Instruction::UDiv, {LHS, RHS},
VPIRFlags::getDefaultFlags(Instruction::UDiv),
DL);
@@ -998,8 +1012,9 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
case scUMaxExpr:
case scSMaxExpr:
case scUMinExpr:
- case scSMinExpr: {
- auto *MinMax = cast<SCEVMinMaxExpr>(S);
+ case scSMinExpr:
+ case scSequentialUMinExpr: {
+ auto *MinMax = cast<SCEVNAryExpr>(S);
Intrinsic::ID IntrinsicID;
switch (S->getSCEVType()) {
case scUMaxExpr:
@@ -1009,6 +1024,7 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
IntrinsicID = Intrinsic::smax;
break;
case scUMinExpr:
+ case scSequentialUMinExpr:
IntrinsicID = Intrinsic::umin;
break;
case scSMinExpr:
@@ -1018,15 +1034,25 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
llvm_unreachable("Unexpected min/max SCEV type");
}
// Chain operands in reverse order matching SCEVExpander's expansion of
- // min/max expressions.
+ // min/max expressions. In SafeUDivMode freeze expansion results of operands
+ // other than the first for sequential UMins, to avoid short-circuiting
+ // divide-by-0/poison.
+ bool IsSequential = S->getSCEVType() == scSequentialUMinExpr;
+ Type *ResultTy = MinMax->getType();
+ bool PrevSafeMode = SafeUDivMode;
SmallVector<VPValue *, 2> Ops;
- for (const SCEVUse &Op : reverse(MinMax->operands())) {
- VPValue *OpV = tryToExpand(Op);
+ for (const SCEV *SCEVOp : reverse(MinMax->operands())) {
+ bool MayShortCircuit =
+ IsSequential && Ops.size() != MinMax->getNumOperands() - 1;
+ SafeUDivMode = MayShortCircuit || PrevSafeMode;
+ VPValue *OpV = tryToExpand(SCEVOp);
+ SafeUDivMode = PrevSafeMode;
if (!OpV)
return nullptr;
+ if (MayShortCircuit)
+ OpV = Builder.createScalarFreeze(OpV, DL);
Ops.push_back(OpV);
}
- Type *ResultTy = MinMax->getType();
VPValue *Result = Ops.front();
for (VPValue *Op : drop_begin(Ops))
Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..60a9a8013de66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -252,6 +252,10 @@ class VPSCEVExpander {
ScalarEvolution &SE;
DebugLoc DL;
+ /// When true, nested SCEVUDivExprs are expanded so that they cannot divide by
+ /// zero, matching SCEVExpander's SafeUDivMode.
+ bool SafeUDivMode = false;
+
/// Try to find a loop-invariant IR value in the plan's entry block whose
/// SCEV matches \p S. Returns the corresponding live-in VPValue, or nullptr
/// if none is found.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 286bf2ebf47f5..2c740670be578 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -449,7 +449,8 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-LABEL: define void @multi_exit(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], i64 [[A:%.*]], i64 [[B:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1)
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[B]], i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[TMP10]], -1
; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]]
; CHECK-NEXT: [[UMIN10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A]])
; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[UMIN10]], 1
More information about the llvm-commits
mailing list