[llvm] [LAA] Bound non-affine monotonic pointer expressions for runtime checks. (PR #210626)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 14:03:12 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/210626
We currently refuse to generate runtime checks for non-affine AddRecs ( e.g. std::bitset-style addresses involving unsigned divides, like words[i/N])
This patch adds getNonAffineMonotonicBounds, whic h , which forms a tight bound for such accesses, if the offset is provably monotonic and non-decreasing over the loop.
It uses the offset value at the first iteration
(SplitIntoInitAndPostInc) and at the last iteration (getSCEVAtScope).
Initially this just supports divides of affine AddRecs, to be extended in the follow-ups.
First step towards fixing https://github.com/llvm/llvm-project/issues/207882.
>From 769e5833972ed5b826990fec245948cf659edf07 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 8 Jul 2026 16:50:16 +0100
Subject: [PATCH] [LAA] Bound non-affine monotonic pointer expressions for
runtime checks.
We currently refuse to generate runtime checks for non-affine AddRecs (
e.g. std::bitset-style addresses involving unsigned divides, like words[i/N])
This patch adds getNonAffineMonotonicBounds, whic h , which forms a tight bound
for such accesses, if the offset is provably monotonic and
non-decreasing over the loop.
It uses the offset value at the first iteration
(SplitIntoInitAndPostInc) and at the last iteration (getSCEVAtScope).
Initially this just supports divides of affine AddRecs, to be extended
in the follow-ups.
First step towards fixing https://github.com/llvm/llvm-project/issues/207882.
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 79 ++++++++++-
.../non-affine-monotonic-bounds.ll | 56 +++++++-
.../non-affine-monotonic-bounds.ll | 124 ++++++++++++++++--
3 files changed, 242 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 8d3c66c8fe321..ac1e974a3f072 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -316,6 +316,67 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(
SE.isKnownPredicate(CmpInst::ICMP_ULE, StartOffset, DerefBytesSCEV);
}
+/// Return true if \p S is known to be monotonically non-decreasing
+/// (in the unsigned sense, without unsigned wrap) across iterations of \p L.
+static bool isKnownNonDecreasingInLoop(const SCEV *S, const Loop *L,
+ ScalarEvolution &SE) {
+ if (SE.isLoopInvariant(S, L))
+ return true;
+
+ switch (S->getSCEVType()) {
+ case scUDivExpr: {
+ // Non-decreasing in the numerator when the divisor is loop-invariant.
+ const auto *UDiv = cast<SCEVUDivExpr>(S);
+ return SE.isLoopInvariant(UDiv->getRHS(), L) &&
+ isKnownNonDecreasingInLoop(UDiv->getLHS(), L, SE);
+ }
+ case scAddRecExpr: {
+ // Non-decreasing if affine for L, without unsigned wrap and non-negative
+ // step.
+ const auto *AR = cast<SCEVAddRecExpr>(S);
+ return AR->getLoop() == L && AR->isAffine() && AR->hasNoUnsignedWrap() &&
+ SE.isKnownNonNegative(AR->getStepRecurrence(SE));
+ }
+ default:
+ return false;
+ }
+}
+
+/// Try to bound a loop-variant pointer that is not an affine AddRec.
+///
+/// If the offset is provably monotonically non-decreasing the accessed range is
+/// bounded by the offset's value at the first iteration (via
+/// SplitIntoInitAndPostInc) and last iteration (via getSCEVAtScope)
+///
+/// Returns {nullptr, nullptr} if no such bound can be formed.
+static std::pair<const SCEV *, const SCEV *>
+getNonAffineMonotonicBounds(const Loop *Lp, const SCEV *PtrExpr,
+ ScalarEvolution *SE) {
+ const auto *PtrAdd = dyn_cast<SCEVAddExpr>(PtrExpr);
+ if (!PtrAdd || !PtrAdd->hasNoUnsignedWrap())
+ return {nullptr, nullptr};
+
+ const SCEV *Base = SE->getPointerBase(PtrExpr);
+ if (isa<SCEVCouldNotCompute>(Base) || !SE->isLoopInvariant(Base, Lp))
+ return {nullptr, nullptr};
+
+ const SCEV *Offset = SE->getMinusSCEV(PtrExpr, Base);
+ if (isa<SCEVCouldNotCompute>(Offset) ||
+ !isKnownNonDecreasingInLoop(Offset, Lp, *SE))
+ return {nullptr, nullptr};
+
+ const SCEV *OffStart = SE->SplitIntoInitAndPostInc(Lp, Offset).first;
+ const SCEV *OffEnd = SE->getSCEVAtScope(Offset, Lp->getParentLoop());
+ if (isa<SCEVCouldNotCompute>(OffStart) || isa<SCEVCouldNotCompute>(OffEnd) ||
+ !SE->isLoopInvariant(OffStart, Lp) || !SE->isLoopInvariant(OffEnd, Lp))
+ return {nullptr, nullptr};
+
+ assert(SE->isKnownPredicate(CmpInst::ICMP_ULE, OffStart, OffEnd) &&
+ "Start must be provably <= End for monotonic expressions");
+
+ return {SE->getAddExpr(Base, OffStart), SE->getAddExpr(Base, OffEnd)};
+}
+
std::pair<const SCEV *, const SCEV *> llvm::getStartAndEndForAccess(
const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, const SCEV *BTC,
const SCEV *MaxBTC, ScalarEvolution *SE,
@@ -396,8 +457,13 @@ std::pair<const SCEV *, const SCEV *> llvm::getStartAndEndForAccess(
ScStart = SE->getUMinExpr(ScStart, ScEnd);
ScEnd = SE->getUMaxExpr(AR->getStart(), ScEnd);
}
- } else
- return {SE->getCouldNotCompute(), SE->getCouldNotCompute()};
+ } else {
+ // The pointer is loop-variant but not an affine AddRec. Try to form a
+ // tight bound for a monotonic offset (see getNonAffineMonotonicBounds).
+ std::tie(ScStart, ScEnd) = getNonAffineMonotonicBounds(Lp, PtrExpr, SE);
+ if (!ScStart)
+ return {SE->getCouldNotCompute(), SE->getCouldNotCompute()};
+ }
assert(SE->isLoopInvariant(ScStart, Lp) && "ScStart needs to be invariant");
assert(SE->isLoopInvariant(ScEnd, Lp) && "ScEnd needs to be invariant");
@@ -1307,8 +1373,13 @@ bool AccessAnalysis::createCheckForAccess(
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(P.getPointer());
if (!AR && Assume)
AR = PSE.getAsAddRec(Ptr, &Predicates);
- if (!AR || !AR->isAffine())
- return false;
+ if (!AR || !AR->isAffine()) {
+ // Check if bounds for non-affine monotonic expressions can be formed.
+ if (!Assume ||
+ !getNonAffineMonotonicBounds(TheLoop, P.getPointer(), SE).first)
+ return false;
+ continue;
+ }
// If there's only one option for Ptr, commit the predicates collected by
// getAsAddRec and look Ptr up again afterwards: the lookup below reads the
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/non-affine-monotonic-bounds.ll b/llvm/test/Analysis/LoopAccessAnalysis/non-affine-monotonic-bounds.ll
index 1b52d94cb1955..c1ea7e48d059f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/non-affine-monotonic-bounds.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/non-affine-monotonic-bounds.ll
@@ -5,10 +5,23 @@
define void @bitset_udiv64_const_tc(ptr %words, ptr %out) {
; CHECK-LABEL: 'bitset_udiv64_const_tc'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Report: cannot identify array bounds
+; CHECK-NEXT: Memory dependences are safe with run-time checks
; CHECK-NEXT: Dependences:
; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.out = getelementptr inbounds i8, ptr %out, i64 %iv
+; CHECK-NEXT: %gep.out = getelementptr inbounds i8, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.words = getelementptr inbounds i8, ptr %words, i64 %div
; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: (441 + %out))
+; CHECK-NEXT: Member: {%out,+,1}<nuw><%loop>
+; CHECK-NEXT: Member: {%out,+,1}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %words High: (7 + %words))
+; CHECK-NEXT: Member: (({0,+,1}<nuw><nsw><%loop> /u 64) + %words)<nuw>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -39,10 +52,23 @@ exit:
define void @bitset_udiv64_symbolic_tc(ptr %words, ptr %out, i64 %N) {
; CHECK-LABEL: 'bitset_udiv64_symbolic_tc'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Report: cannot identify array bounds
+; CHECK-NEXT: Memory dependences are safe with run-time checks
; CHECK-NEXT: Dependences:
; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.out = getelementptr inbounds i8, ptr %out, i64 %iv
+; CHECK-NEXT: %gep.out = getelementptr inbounds i8, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.words = getelementptr inbounds i8, ptr %words, i64 %div
; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: (%N + %out))
+; CHECK-NEXT: Member: {%out,+,1}<nuw><%loop>
+; CHECK-NEXT: Member: {%out,+,1}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %words High: (1 + ((-1 + %N) /u 64) + %words))
+; CHECK-NEXT: Member: (({0,+,1}<nuw><nsw><%loop> /u 64) + %words)<nuw>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -73,10 +99,21 @@ exit:
define void @udiv3_symbolic_tc(ptr %a, ptr %b, i64 %N) {
; CHECK-LABEL: 'udiv3_symbolic_tc'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Report: cannot identify array bounds
+; CHECK-NEXT: Memory dependences are safe with run-time checks
; CHECK-NEXT: Dependences:
; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.a = getelementptr inbounds i8, ptr %a, i64 %div
; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %b High: (%N + %b))
+; CHECK-NEXT: Member: {%b,+,1}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %a High: (1 + ((-1 + %N) /u 3) + %a))
+; CHECK-NEXT: Member: (({0,+,1}<nuw><nsw><%loop> /u 3) + %a)<nuw>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -105,10 +142,21 @@ exit:
define void @udiv_invariant_divisor(ptr %a, ptr %b, i64 %N, i64 %d) {
; CHECK-LABEL: 'udiv_invariant_divisor'
; CHECK-NEXT: loop:
-; CHECK-NEXT: Report: cannot identify array bounds
+; CHECK-NEXT: Memory dependences are safe with run-time checks
; CHECK-NEXT: Dependences:
; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.a = getelementptr inbounds i8, ptr %a, i64 %div
; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %b High: (%N + %b))
+; CHECK-NEXT: Member: {%b,+,1}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %a High: (1 + ((-1 + %N) /u %d) + %a))
+; CHECK-NEXT: Member: (({0,+,1}<nuw><nsw><%loop> /u %d) + %a)<nuw>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
diff --git a/llvm/test/Transforms/LoopVectorize/non-affine-monotonic-bounds.ll b/llvm/test/Transforms/LoopVectorize/non-affine-monotonic-bounds.ll
index 545f10b457ec1..9fa09c0a79d0d 100644
--- a/llvm/test/Transforms/LoopVectorize/non-affine-monotonic-bounds.ll
+++ b/llvm/test/Transforms/LoopVectorize/non-affine-monotonic-bounds.ll
@@ -5,10 +5,43 @@
define void @bitset_copy_symbolic_tc(ptr %src, ptr %out, i64 %N) {
; CHECK-LABEL: define void @bitset_copy_symbolic_tc(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[OUT:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 [[N]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 6
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP5]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[DIV:%.*]] = lshr i64 [[IV]], 6
; CHECK-NEXT: [[GEP_WORDS:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[DIV]]
; CHECK-NEXT: [[W:%.*]] = load i8, ptr [[GEP_WORDS]], align 1
@@ -16,7 +49,7 @@ define void @bitset_copy_symbolic_tc(ptr %src, ptr %out, i64 %N) {
; CHECK-NEXT: store i8 [[W]], ptr [[GEP_OUT]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -42,10 +75,36 @@ exit:
define void @bitset_copy_const_tc(ptr %src, ptr %out) {
; CHECK-LABEL: define void @bitset_copy_const_tc(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[OUT:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 441
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 7
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[INDEX]], 6
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP1]], align 1, !alias.scope [[META9:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 1, !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 440
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 440, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[DIV:%.*]] = lshr i64 [[IV]], 6
; CHECK-NEXT: [[GEP_WORDS:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[DIV]]
; CHECK-NEXT: [[W:%.*]] = load i8, ptr [[GEP_WORDS]], align 1
@@ -53,7 +112,7 @@ define void @bitset_copy_const_tc(ptr %src, ptr %out) {
; CHECK-NEXT: store i8 [[W]], ptr [[GEP_OUT]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 441
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -192,10 +251,57 @@ exit:
define void @udiv3_copy(ptr %a, ptr %b, i64 %N) {
; CHECK-LABEL: define void @udiv3_copy(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[N]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[TMP5]], align 1, !alias.scope [[META16:![0-9]+]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP7]], align 1, !alias.scope [[META16]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP9]], align 1, !alias.scope [[META16]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1, !alias.scope [[META16]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i8> poison, i8 [[TMP12]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i8> [[TMP16]], i8 [[TMP13]], i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP17]], i8 [[TMP14]], i32 2
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> [[TMP18]], i8 [[TMP15]], i32 3
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[TMP19]], ptr [[TMP20]], align 1, !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[IV]], 3
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DIV]]
; CHECK-NEXT: [[LA:%.*]] = load i8, ptr [[GEP_A]], align 1
@@ -203,7 +309,7 @@ define void @udiv3_copy(ptr %a, ptr %b, i64 %N) {
; CHECK-NEXT: store i8 [[LA]], ptr [[GEP_B]], align 1
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list