[llvm] fc9f8ec - [VPlan] Expand sequential/regular UMin SCEVs in VPSCEVExpander. (#209786)

via llvm-commits llvm-commits at lists.llvm.org
Sat Aug 22 12:18:24 PDT 2026


Author: Florian Hahn
Date: 2026-08-22T19:18:19Z
New Revision: fc9f8ec195b9dd29faaf73cb3dfe058df3ddd63d

URL: https://github.com/llvm/llvm-project/commit/fc9f8ec195b9dd29faaf73cb3dfe058df3ddd63d
DIFF: https://github.com/llvm/llvm-project/commit/fc9f8ec195b9dd29faaf73cb3dfe058df3ddd63d.diff

LOG: [VPlan] Expand sequential/regular UMin SCEVs in VPSCEVExpander. (#209786)

Add support for expanding SequentialUMinExpr SCEV expressions in
VPSCEVExpander.

For regular UMin expressions, the expansion unconditionally expands &
executes all operands, while the semantics of sequential UMin only
require the first operand to be evaluated unconditionally.

For sequential UMin expressions, we need to make sure potentially
UB/poison generating operands must be accounted for. Matching IR SCEV
expander, make sure that divisors of UDiv are poison-free and non-zero
inside sequential UMin. Similarly, freeze all operands other than the
first, to avoid poison from propagating.

PR: https://github.com/llvm/llvm-project/pull/209786

Added: 
    

Modified: 
    llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
    llvm/lib/Transforms/Vectorize/VPlanUtils.h
    llvm/test/Transforms/LoopVectorize/X86/cost-model.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e460f63953ac4..e6b267493d987 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -945,9 +945,23 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
     VPValue *LHS = tryToExpand(UDiv->getLHS());
     if (!LHS)
       return nullptr;
-    VPValue *RHS = tryToExpand(UDiv->getRHS());
+    const SCEV *RHSExpr = UDiv->getRHS();
+    VPValue *RHS = tryToExpand(RHSExpr);
     if (!RHS)
       return nullptr;
+    if (SafeUDivMode) {
+      // Make sure the UDiv's divisor is guaranteed to not be zero/poison, to
+      // avoid UB.
+      Type *Ty = UDiv->getType();
+      bool GuaranteedNotPoison =
+          ScalarEvolution::isGuaranteedNotToBePoison(RHSExpr);
+      if (!GuaranteedNotPoison)
+        RHS = Builder.createScalarFreeze(RHS, DL);
+      if (!SE.isKnownNonZero(RHSExpr) || !GuaranteedNotPoison)
+        RHS = Builder.createScalarIntrinsic(
+            Intrinsic::umax, {RHS, Builder.getPlan().getConstantInt(Ty, 1)}, Ty,
+            DL);
+    }
     return Builder.createNaryOp(Instruction::UDiv, {LHS, RHS},
                                 VPIRFlags::getDefaultFlags(Instruction::UDiv),
                                 DL);
@@ -998,8 +1012,9 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
   case scUMaxExpr:
   case scSMaxExpr:
   case scUMinExpr:
-  case scSMinExpr: {
-    auto *MinMax = cast<SCEVMinMaxExpr>(S);
+  case scSMinExpr:
+  case scSequentialUMinExpr: {
+    auto *MinMax = cast<SCEVNAryExpr>(S);
     Intrinsic::ID IntrinsicID;
     switch (S->getSCEVType()) {
     case scUMaxExpr:
@@ -1009,6 +1024,7 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
       IntrinsicID = Intrinsic::smax;
       break;
     case scUMinExpr:
+    case scSequentialUMinExpr:
       IntrinsicID = Intrinsic::umin;
       break;
     case scSMinExpr:
@@ -1018,15 +1034,25 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
       llvm_unreachable("Unexpected min/max SCEV type");
     }
     // Chain operands in reverse order matching SCEVExpander's expansion of
-    // min/max expressions.
+    // min/max expressions. In SafeUDivMode freeze expansion results of operands
+    // other than the first for sequential UMins, to avoid short-circuiting
+    // divide-by-0/poison.
+    bool IsSequential = S->getSCEVType() == scSequentialUMinExpr;
+    Type *ResultTy = MinMax->getType();
+    bool PrevSafeMode = SafeUDivMode;
     SmallVector<VPValue *, 2> Ops;
-    for (const SCEVUse &Op : reverse(MinMax->operands())) {
-      VPValue *OpV = tryToExpand(Op);
+    for (const SCEV *SCEVOp : reverse(MinMax->operands())) {
+      bool MayShortCircuit =
+          IsSequential && Ops.size() != MinMax->getNumOperands() - 1;
+      SafeUDivMode = MayShortCircuit || PrevSafeMode;
+      VPValue *OpV = tryToExpand(SCEVOp);
+      SafeUDivMode = PrevSafeMode;
       if (!OpV)
         return nullptr;
+      if (MayShortCircuit)
+        OpV = Builder.createScalarFreeze(OpV, DL);
       Ops.push_back(OpV);
     }
-    Type *ResultTy = MinMax->getType();
     VPValue *Result = Ops.front();
     for (VPValue *Op : drop_begin(Ops))
       Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},

diff  --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..60a9a8013de66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -252,6 +252,10 @@ class VPSCEVExpander {
   ScalarEvolution &SE;
   DebugLoc DL;
 
+  /// When true, nested SCEVUDivExprs are expanded so that they cannot divide by
+  /// zero, matching SCEVExpander's SafeUDivMode.
+  bool SafeUDivMode = false;
+
   /// Try to find a loop-invariant IR value in the plan's entry block whose
   /// SCEV matches \p S. Returns the corresponding live-in VPValue, or nullptr
   /// if none is found.

diff  --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 286bf2ebf47f5..2c740670be578 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -449,7 +449,8 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
 ; CHECK-LABEL: define void @multi_exit(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], i64 [[A:%.*]], i64 [[B:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1)
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[B]], i64 1)
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[TMP10]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[TMP0]]
 ; CHECK-NEXT:    [[UMIN10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i64 [[UMIN10]], 1


        


More information about the llvm-commits mailing list