[llvm] [VPlan] Support pointer min/max bounds in VPlan memory runtime checks. (PR #225828)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 05:38:37 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/225828
>From 4499796c17b9647d7315126808253e77db3247c9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 23 Sep 2026 10:41:15 +0100
Subject: [PATCH 1/2] [VPlan] Support pointer min/max bounds in VPlan memory
runtime checks.
Expand pointer-typed SCEV min/max expressions in VPSCEVExpander as
icmp + select (including profile metadata), matching SCEVExpander.
This allows modeling memory runtime checks with pointer min/max bounds
in VPlan, e.g. for accesses with a runtime stride of unknown sign.
Depends on https://github.com/llvm/llvm-project/pull/221483 (included in
PR)
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 21 +++++++--
.../LoopVectorize/VPlan/memory-checks.ll | 46 +++++++++----------
.../LoopVectorize/scev-check-unknown-prof.ll | 32 ++++++-------
5 files changed, 64 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7b0e80390379f4..3409e8b86a9b91 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7032,12 +7032,8 @@ void LoopVectorizationPlanner::attachRuntimeChecks(
"(e.g., adding 'restrict').";
});
}
- // VPSCEVExpander expands AddRecs in the plan's entry, not the check block,
- // and does not support pointer-typed min/max yet.
- auto IsUnsupported = [](const SCEV *S) {
- return isa<SCEVAddRecExpr>(S) ||
- (isa<SCEVMinMaxExpr>(S) && S->getType()->isPointerTy());
- };
+ // VPSCEVExpander expands AddRecs in the plan's entry, not the check block.
+ auto IsUnsupported = [](const SCEV *S) { return isa<SCEVAddRecExpr>(S); };
// Diff checks are not modelled in VPlan yet, and the VPlan expander cannot
// hoist bounds out of an enclosing loop.
const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2a6349dd670db5..99bf690d8e6e7e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -809,8 +809,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
- return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
- Name);
+ Value *Sel =
+ Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(), Name);
+ if (auto *I = dyn_cast<Instruction>(Sel))
+ applyMetadata(*I);
+ return Sel;
}
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask: {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index af12343e8a74b6..3719923cfd93d1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1102,9 +1102,24 @@ VPValue *VPSCEVExpander::expand(const SCEV *S) {
Ops.push_back(OpV);
}
VPValue *Result = Ops.front();
- for (VPValue *Op : drop_begin(Ops))
- Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
- ResultTy, DL);
+ for (VPValue *Op : drop_begin(Ops)) {
+ if (!ResultTy->isPointerTy()) {
+ Result = Builder.createScalarIntrinsic(IntrinsicID, {Result, Op},
+ ResultTy, DL);
+ continue;
+ }
+ // The min/max intrinsics don't support pointer operands, so expand
+ // pointer-typed min/max as cmp + select, matching SCEVExpander.
+ VPValue *Cmp = Builder.createICmp(
+ MinMaxIntrinsic::getPredicate(IntrinsicID), Result, Op, DL);
+ VPInstruction *Sel = Builder.createSelect(Cmp, Result, Op, DL);
+ Function &F =
+ *Builder.getPlan().getScalarHeader()->getIRBasicBlock()->getParent();
+ if (MDNode *MD =
+ getExplicitlyUnknownBranchWeightsIfProfiled(F, "scev-expander"))
+ Sel->setMetadata(LLVMContext::MD_prof, MD);
+ Result = Sel;
+ }
return Result;
}
case scAddRecExpr: {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll b/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll
index 2389a5434dfe1c..f52a8edeacef99 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll
@@ -67,33 +67,33 @@ define void @ptr_minmax_bounds(ptr %a, ptr %b, i64 %n, i64 %s, i64 %t) {
; CHECK-NEXT: IR %step = mul i64 %s, %t
; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%n>, ir<4>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
-; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
+; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.memcheck
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<vector.memcheck>:
-; CHECK-NEXT: IR %0 = shl i64 %n, 2
-; CHECK-NEXT: IR %scevgep = getelementptr i8, ptr %b, i64 %0
-; CHECK-NEXT: IR %1 = mul i64 %t, %s
-; CHECK-NEXT: IR %2 = add i64 %n, -1
-; CHECK-NEXT: IR %3 = mul i64 %1, %2
-; CHECK-NEXT: IR %4 = shl i64 %3, 2
-; CHECK-NEXT: IR %scevgep1 = getelementptr i8, ptr %a, i64 %4
-; CHECK-NEXT: IR %5 = icmp ult ptr %a, %scevgep1
-; CHECK-NEXT: IR %umin = select i1 %5, ptr %a, ptr %scevgep1
-; CHECK-NEXT: IR %6 = icmp ugt ptr %a, %scevgep1
-; CHECK-NEXT: IR %umax = select i1 %6, ptr %a, ptr %scevgep1
-; CHECK-NEXT: IR %scevgep2 = getelementptr i8, ptr %umax, i64 4
-; CHECK-NEXT: IR %bound0 = icmp ult ptr %b, %scevgep2
-; CHECK-NEXT: IR %bound1 = icmp ult ptr %umin, %scevgep
-; CHECK-NEXT: IR %found.conflict = and i1 %bound0, %bound1
-; CHECK-NEXT: EMIT branch-on-cond ir<%found.conflict>
+; CHECK-NEXT: vector.memcheck:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = shl ir<%n>, ir<2>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = ptradd ir<%b>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = add ir<%n>, ir<-1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = mul ir<%t>, ir<%s>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = mul vp<[[VP6]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = shl vp<[[VP7]]>, ir<2>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = ptradd ir<%a>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp ult ir<%a>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = select vp<[[VP10]]>, ir<%a>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ugt ir<%a>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = select vp<[[VP12]]>, ir<%a>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = ptradd vp<[[VP13]]>, ir<4>
+; CHECK-NEXT: EMIT vp<%bound0> = icmp ult ir<%b>, vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<%bound1> = icmp ult vp<[[VP11]]>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<%found.conflict> = and vp<%bound0>, vp<%bound1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%found.conflict>
; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = and ir<%n>, ir<3>
-; CHECK-NEXT: EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = broadcast ir<%step>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = step-vector i64
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = broadcast ir<4>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = and ir<%n>, ir<3>
+; CHECK-NEXT: EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP16]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = broadcast ir<%step>
+; CHECK-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = step-vector i64
+; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = broadcast ir<4>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/scev-check-unknown-prof.ll b/llvm/test/Transforms/LoopVectorize/scev-check-unknown-prof.ll
index f2d2f13a39a6bf..05ef106661e4c3 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-check-unknown-prof.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-check-unknown-prof.ll
@@ -16,12 +16,12 @@ define void @wrap_check(i32 %n, i32 %step) !prof !0 {
; CHECK: br i1 [[TMP16:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK: [[VECTOR_BODY:.*]]:
-; CHECK: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK: br i1 [[CMP_N:%.*]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK: [[LOOP:.*]]:
-; CHECK: br i1 [[EC:%.*]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: br i1 [[EC:%.*]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK: [[EXIT]]:
;
@@ -61,17 +61,17 @@ define void @runtime_step_memcheck(ptr %in, ptr %out, i64 %n, i64 %step) !prof !
; CHECK: [[TMP9:%.*]] = select i1 [[TMP3]], i1 [[TMP8:%.*]], i1 [[TMP7:%.*]], !prof [[PROF1]]
; CHECK: br i1 [[TMP16:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK: [[UMIN:%.*]] = select i1 [[TMP23:%.*]], ptr [[IN]], ptr [[SCEVGEP1:%.*]], !prof [[PROF1]]
-; CHECK: [[UMAX:%.*]] = select i1 [[TMP24:%.*]], ptr [[IN]], ptr [[SCEVGEP1]], !prof [[PROF1]]
+; CHECK: [[TMP26:%.*]] = select i1 [[TMP25:%.*]], ptr [[IN]], ptr [[TMP24:%.*]], !prof [[PROF1]]
+; CHECK: [[TMP28:%.*]] = select i1 [[TMP27:%.*]], ptr [[IN]], ptr [[TMP24]], !prof [[PROF1]]
; CHECK: br i1 [[FOUND_CONFLICT:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK: [[VECTOR_BODY:.*]]:
-; CHECK: br i1 [[TMP47:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: br i1 [[TMP52:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK: br i1 [[CMP_N:%.*]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK: [[LOOP:.*]]:
-; CHECK: br i1 [[EC:%.*]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: br i1 [[EC:%.*]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK: [[EXIT]]:
;
@@ -113,12 +113,12 @@ define void @wrap_check_not_profiled(i32 %n, i32 %step) {
; CHECK: br i1 [[TMP16:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK: [[VECTOR_BODY:.*]]:
-; CHECK: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK: br i1 [[CMP_N:%.*]], label %[[EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK: [[LOOP:.*]]:
-; CHECK: br i1 [[EC:%.*]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: br i1 [[EC:%.*]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK: [[EXIT]]:
;
@@ -150,12 +150,12 @@ exit:
;.
; CHECK: [[PROF0]] = !{!"function_entry_count", i64 1000}
; CHECK: [[PROF1]] = !{!"unknown", !"scev-expander"}
-; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
-; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META3]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META2]], [[META3]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]]}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META4]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]]}
;.
>From c9d4045b544a3459723e4aea8cdedd28136c1b04 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 2 Oct 2026 13:28:20 +0100
Subject: [PATCH 2/2] !fixup addressi latest comments
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e287d2b59d4308..9e674d204d1ac1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6966,7 +6966,7 @@ void LoopVectorizationPlanner::attachRuntimeChecks(
});
}
// VPSCEVExpander expands AddRecs in the plan's entry, not the check block.
- auto IsUnsupported = [](const SCEV *S) { return isa<SCEVAddRecExpr>(S); };
+ auto IsUnsupported = IsaPred<SCEVAddRecExpr>;
// Diff checks are not modelled in VPlan yet, and the VPlan expander cannot
// hoist bounds out of an enclosing loop.
const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index cff82c1caa0065..8e626e164d8875 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1112,13 +1112,12 @@ VPValue *VPSCEVExpander::expand(const SCEV *S) {
// pointer-typed min/max as cmp + select, matching SCEVExpander.
VPValue *Cmp = Builder.createICmp(
MinMaxIntrinsic::getPredicate(IntrinsicID), Result, Op, DL);
- VPInstruction *Sel = Builder.createSelect(Cmp, Result, Op, DL);
+ Result = Builder.createSelect(Cmp, Result, Op, DL);
Function &F =
*Builder.getPlan().getScalarHeader()->getIRBasicBlock()->getParent();
if (MDNode *MD =
getExplicitlyUnknownBranchWeightsIfProfiled(F, "scev-expander"))
- Sel->setMetadata(LLVMContext::MD_prof, MD);
- Result = Sel;
+ cast<VPInstruction>(Result)->setMetadata(LLVMContext::MD_prof, MD);
}
return Result;
}
More information about the llvm-commits
mailing list