[llvm] [LAA] Fix off-by-EltSize in negative-step deref bounds check (PR #211964)
Aleksandr Popov via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 04:26:13 PDT 2026
https://github.com/aleks-tmb updated https://github.com/llvm/llvm-project/pull/211964
>From 78cfe4082f70ec580b8de17d2cd55846a7b79e27 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Mon, 10 Aug 2026 21:19:15 +0000
Subject: [PATCH 1/3] [LoopVectorize] Add tests for reverse early-exit loops in
deref regions
Related to #211962
---
.../LoopVectorize/reverse-loop-length.ll | 97 +++++++++++++++++++
1 file changed, 97 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
diff --git a/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll b/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
new file mode 100644
index 0000000000000..ec9c3ffea70e5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+; The loops below walk a dereferenceable region backwards, so every load is
+; safe.
+
+; TODO: The AR runs over [0, %length), which is exactly the region the assume
+; marks dereferenceable, so the early-exit loop should be vectorized.
+define ptr @reverse_reaches_base_dynamic_length(i64 %length, ptr %ptr) {
+; CHECK-LABEL: define ptr @reverse_reaches_base_dynamic_length(
+; CHECK-SAME: i64 [[LENGTH:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[NULL_CHECK:%.*]] = icmp eq i64 [[LENGTH]], 0
+; CHECK-NEXT: br i1 [[NULL_CHECK]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[PTR]], i64 [[LENGTH]]) ]
+; CHECK-NEXT: [[START:%.*]] = sub i64 [[LENGTH]], 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ], [ [[START]], %[[PREHEADER]] ]
+; CHECK-NEXT: [[ELEMENT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[ELEMENT_GEP]], align 1
+; CHECK-NEXT: [[FOUND_CHECK:%.*]] = icmp eq i8 [[ELEMENT]], 0
+; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT_LOOPEXIT:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
+; CHECK-NEXT: [[RANGE_CHECK:%.*]] = icmp ne i64 [[IV]], 0
+; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret ptr null
+;
+entry:
+ %null_check = icmp eq i64 %length, 0
+ br i1 %null_check, label %exit, label %preheader
+
+preheader:
+ call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %ptr, i64 %length) ]
+ %start = sub i64 %length, 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %iv.next, %latch ], [ %start, %preheader ]
+ %element_gep = getelementptr i8, ptr %ptr, i64 %iv
+ %element = load i8, ptr %element_gep, align 1
+ %found_check = icmp eq i8 %element, 0
+ br i1 %found_check, label %exit, label %latch
+
+latch:
+ %iv.next = add i64 %iv, -1
+ %range_check = icmp ne i64 %iv, 0
+ br i1 %range_check, label %loop, label %exit
+
+exit:
+ ret ptr null
+}
+
+; TODO: The last iteration lands exactly on %ptr, so the accessed range [0, 16)
+; exactly fills the dereferenceable region, and the early-exit loop should be
+; vectorized.
+define ptr @reverse_reaches_base_static_length(ptr dereferenceable(16) %ptr) {
+; CHECK-LABEL: define ptr @reverse_reaches_base_static_length(
+; CHECK-SAME: ptr dereferenceable(16) [[PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ], [ 15, %[[ENTRY]] ]
+; CHECK-NEXT: [[ELEMENT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[IV]]
+; CHECK-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[ELEMENT_GEP]], align 1
+; CHECK-NEXT: [[FOUND_CHECK:%.*]] = icmp eq i8 [[ELEMENT]], 0
+; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
+; CHECK-NEXT: [[RANGE_CHECK:%.*]] = icmp ne i64 [[IV]], 0
+; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret ptr null
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %iv.next, %latch ], [ 15, %entry ]
+ %element_gep = getelementptr i8, ptr %ptr, i64 %iv
+ %element = load i8, ptr %element_gep, align 1
+ %found_check = icmp eq i8 %element, 0
+ br i1 %found_check, label %exit, label %latch
+
+latch:
+ %iv.next = add i64 %iv, -1
+ %range_check = icmp ne i64 %iv, 0
+ br i1 %range_check, label %loop, label %exit
+
+exit:
+ ret ptr null
+}
>From 3018408c90fa9856a897bd9e37e7602e078686bc Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Thu, 1 Oct 2026 15:15:55 +0000
Subject: [PATCH 2/3] Apply review suggestions
---
.../LoopVectorize/reverse-loop-length.ll | 67 ++++++++++---------
1 file changed, 36 insertions(+), 31 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll b/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
index ec9c3ffea70e5..dbaaeadb6a890 100644
--- a/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
+++ b/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
@@ -4,32 +4,34 @@
; The loops below walk a dereferenceable region backwards, so every load is
; safe.
-; TODO: The AR runs over [0, %length), which is exactly the region the assume
-; marks dereferenceable, so the early-exit loop should be vectorized.
+; TODO: The counting-down IV accesses [0, %length), which is exactly the region
+; the assume marks dereferenceable, so the early-exit loop should be vectorized.
define ptr @reverse_reaches_base_dynamic_length(i64 %length, ptr %ptr) {
; CHECK-LABEL: define ptr @reverse_reaches_base_dynamic_length(
; CHECK-SAME: i64 [[LENGTH:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[NULL_CHECK:%.*]] = icmp eq i64 [[LENGTH]], 0
; CHECK-NEXT: br i1 [[NULL_CHECK]], label %[[EXIT:.*]], label %[[PREHEADER:.*]]
; CHECK: [[PREHEADER]]:
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[PTR]], i64 [[LENGTH]]) ]
; CHECK-NEXT: [[START:%.*]] = sub i64 [[LENGTH]], 1
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ], [ [[START]], %[[PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[START]], %[[PREHEADER]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[ELEMENT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[IV]]
; CHECK-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[ELEMENT_GEP]], align 1
; CHECK-NEXT: [[FOUND_CHECK:%.*]] = icmp eq i8 [[ELEMENT]], 0
-; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT_LOOPEXIT:.*]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
+; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
; CHECK-NEXT: [[RANGE_CHECK:%.*]] = icmp ne i64 [[IV]], 0
-; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]]
+; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP_HEADER]], label %[[EXIT_LOOPEXIT]]
; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[RES_PH:%.*]] = phi ptr [ null, %[[LOOP_LATCH]] ], [ [[ELEMENT_GEP]], %[[LOOP_HEADER]] ]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret ptr null
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[RES_PH]], %[[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT: ret ptr [[RES]]
;
entry:
%null_check = icmp eq i64 %length, 0
@@ -38,22 +40,23 @@ entry:
preheader:
call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %ptr, i64 %length) ]
%start = sub i64 %length, 1
- br label %loop
+ br label %loop.header
-loop:
- %iv = phi i64 [ %iv.next, %latch ], [ %start, %preheader ]
+loop.header:
+ %iv = phi i64 [ %start, %preheader ], [ %iv.next, %loop.latch ]
%element_gep = getelementptr i8, ptr %ptr, i64 %iv
%element = load i8, ptr %element_gep, align 1
%found_check = icmp eq i8 %element, 0
- br i1 %found_check, label %exit, label %latch
+ br i1 %found_check, label %exit, label %loop.latch
-latch:
+loop.latch:
%iv.next = add i64 %iv, -1
%range_check = icmp ne i64 %iv, 0
- br i1 %range_check, label %loop, label %exit
+ br i1 %range_check, label %loop.header, label %exit
exit:
- ret ptr null
+ %res = phi ptr [ null, %entry ], [ %element_gep, %loop.header ], [ null, %loop.latch ]
+ ret ptr %res
}
; TODO: The last iteration lands exactly on %ptr, so the accessed range [0, 16)
@@ -63,35 +66,37 @@ define ptr @reverse_reaches_base_static_length(ptr dereferenceable(16) %ptr) {
; CHECK-LABEL: define ptr @reverse_reaches_base_static_length(
; CHECK-SAME: ptr dereferenceable(16) [[PTR:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ], [ 15, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 15, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[ELEMENT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[IV]]
; CHECK-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[ELEMENT_GEP]], align 1
; CHECK-NEXT: [[FOUND_CHECK:%.*]] = icmp eq i8 [[ELEMENT]], 0
-; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT:.*]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
+; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
; CHECK-NEXT: [[RANGE_CHECK:%.*]] = icmp ne i64 [[IV]], 0
-; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP]], label %[[EXIT]]
+; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP_HEADER]], label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret ptr null
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[ELEMENT_GEP]], %[[LOOP_HEADER]] ], [ null, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: ret ptr [[RES]]
;
entry:
- br label %loop
+ br label %loop.header
-loop:
- %iv = phi i64 [ %iv.next, %latch ], [ 15, %entry ]
+loop.header:
+ %iv = phi i64 [ 15, %entry ], [ %iv.next, %loop.latch ]
%element_gep = getelementptr i8, ptr %ptr, i64 %iv
%element = load i8, ptr %element_gep, align 1
%found_check = icmp eq i8 %element, 0
- br i1 %found_check, label %exit, label %latch
+ br i1 %found_check, label %exit, label %loop.latch
-latch:
+loop.latch:
%iv.next = add i64 %iv, -1
%range_check = icmp ne i64 %iv, 0
- br i1 %range_check, label %loop, label %exit
+ br i1 %range_check, label %loop.header, label %exit
exit:
- ret ptr null
+ %res = phi ptr [ %element_gep, %loop.header ], [ null, %loop.latch ]
+ ret ptr %res
}
>From 14c9f972d0106238f8165625d26d0844890c1627 Mon Sep 17 00:00:00 2001
From: Aleksandr Popov <apopov at azul.com>
Date: Mon, 27 Jul 2026 00:08:52 +0000
Subject: [PATCH 3/3] [LAA] Fix off-by-EltSize bounds for negative-step deref
checks
evaluatePtrAddRecAtMaxBTCWillNotWrap treated AR->getStart() as the
lowest accessed address. For a negative step that is the *highest*
one, so both safety checks on the negative-step branch were off by
EltSize: the lower bound was over-strict, rejecting loops whose last
iteration lands exactly on the base pointer, and the upper bound
under-counted the top access, accepting loops that read past
DerefBytes.
Compute the lowest address per direction - AR->getStart() for a
non-negative step, AR->evaluateAtIteration(MaxBTC, SE) otherwise -
so both checks share one expression and the per-direction MaxOffset
branch goes away.
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 56 +++++++++----------
.../negative-step-deref-off-by-eltsize.ll | 12 ++--
.../LoopVectorize/reverse-loop-length.ll | 45 ++++++++++-----
3 files changed, 64 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 9c77a78336de2..5b04642951c14 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -206,6 +206,18 @@ static const SCEV *mulSCEVNoOverflow(const SCEV *A, const SCEV *B,
/// Return true, if evaluating \p AR at \p MaxBTC cannot wrap, because \p AR at
/// \p MaxBTC is guaranteed inbounds of the accessed object.
+///
+/// StartPtr is the pointer base of \p AR's start value, DerefBytes the number
+/// of bytes known to be dereferenceable from it and Step \p AR's step
+/// recurrence. The accessed byte range is
+/// [LowestOffset, LowestOffset + AccessedBytes), where
+/// AccessedBytes = \p MaxBTC * |Step| + \p EltSize,
+/// LowestOffset = smallest byte offset from StartPtr any iteration reaches.
+///
+/// The function returns true only when both safety invariants hold, regardless
+/// of step direction:
+/// 1. LowestOffset >= 0 (no access below StartPtr)
+/// 2. LowestOffset + AccessedBytes <= DerefBytes (no access past the region)
static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(
const SCEVAddRecExpr *AR, const SCEV *MaxBTC, const SCEV *EltSize,
ScalarEvolution &SE, const DataLayout &DL, DominatorTree *DT,
@@ -264,15 +276,15 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(
Step = SE.getNoopOrSignExtend(Step, WiderTy);
MaxBTC = SE.getNoopOrZeroExtend(MaxBTC, WiderTy);
- // For the computations below, make sure they don't unsigned wrap.
- // FIXME: for a negative step the lowest accessed address is not
- // AR->getStart() but AR->evaluateAtIteration(MaxBTC, SE); the check below
- // therefore compares StartPtr against the highest accessed address instead
- // of the lowest.
- if (!SE.isKnownPredicate(CmpInst::ICMP_UGE, AR->getStart(), StartPtr))
+ SCEVUse LowestAddr = AR->getStart();
+ if (!IsKnownNonNegative)
+ LowestAddr = AR->evaluateAtIteration(MaxBTC, SE);
+ // Lower-bound safety check: the lowest accessed address must not fall below
+ // StartPtr.
+ if (!SE.isKnownPredicate(CmpInst::ICMP_UGE, LowestAddr, StartPtr))
return false;
- const SCEV *StartOffset = SE.getNoopOrZeroExtend(
- SE.getMinusSCEV(AR->getStart(), StartPtr), WiderTy);
+ const SCEV *LowestOffset =
+ SE.getNoopOrZeroExtend(SE.getMinusSCEV(LowestAddr, StartPtr), WiderTy);
if (!LoopGuards)
LoopGuards.emplace(ScalarEvolution::LoopGuards::collect(AR->getLoop(), SE));
@@ -301,27 +313,15 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(
if (!AccessedBytes)
return false;
- // Compute MaxOffset per direction: exclusive upper offset of the
- // accessed range.
- const SCEV *MaxOffset;
- if (IsKnownNonNegative) {
- MaxOffset = addSCEVNoOverflow(StartOffset, AccessedBytes, SE);
- if (!MaxOffset)
- return false;
+ // Exclusive upper offset of the accessed range.
+ const SCEV *MaxOffset = addSCEVNoOverflow(LowestOffset, AccessedBytes, SE);
+ if (!MaxOffset)
+ return false;
+ // FIXME: Applying the guards unconditionally would also help reverse loops
+ // with a symbolic deref region; will be done separately.
+ if (IsKnownNonNegative)
DerefBytesSCEV = SE.applyLoopGuards(DerefBytesSCEV, *LoopGuards);
- } else {
- // FIXME: two independent off-by-EltSize bugs on this branch:
- // 1. StartOffset here is actually the HIGHEST offset, because it is
- // computed from AR->getStart() rather than
- // AR->evaluateAtIteration(MaxBTC, SE) (see FIXME above).
- // 2. The lower check is over-strict by EltSize and the upper is
- // under-counted by EltSize.
- assert(SE.isKnownNegative(Step) && "must be known negative");
- if (!SE.isKnownPredicate(CmpInst::ICMP_SGE, StartOffset, AccessedBytes))
- return false;
- MaxOffset = StartOffset;
- }
- // MaxOffset must not exceed the deref-region end.
+ // Upper-bound safety check: MaxOffset must not exceed the deref-region end.
return SE.isKnownPredicate(CmpInst::ICMP_ULE, MaxOffset, DerefBytesSCEV);
}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/negative-step-deref-off-by-eltsize.ll b/llvm/test/Analysis/LoopAccessAnalysis/negative-step-deref-off-by-eltsize.ll
index b9f1857e40137..ec1d790c6a664 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/negative-step-deref-off-by-eltsize.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/negative-step-deref-off-by-eltsize.ll
@@ -4,7 +4,7 @@
; Reverse loop loading 4 i32 elements whose access range exactly fills the
; dereferenceable region (deref(16), reads bytes [0, 16)).
;
-; TODO: LAA should recognise that this AR fits within the deref
+; LAA should recognise that this AR fits within the deref
; region and produce tight bounds (Low: %A, High: %A + 16).
;
; Pseudocode:
@@ -28,10 +28,10 @@ define void @reverse_reaches_base(ptr dereferenceable(16) %A, ptr dereferenceabl
; CHECK-NEXT: %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: null High: (16 + %B)<nuw>)
+; CHECK-NEXT: (Low: %B High: (16 + %B)<nuw>)
; CHECK-NEXT: Member: {(12 + %B)<nuw>,+,-4}<nw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: null High: (16 + %A)<nuw>)
+; CHECK-NEXT: (Low: %A High: (16 + %A)<nuw>)
; CHECK-NEXT: Member: {(12 + %A)<nuw>,+,-4}<nw><%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
@@ -67,7 +67,7 @@ exit.done:
; The top i32 read at byte 13 covers [13, 17), but deref(16) only
; guarantees [0, 16) — bytes at/after 16 may or may not be dereferenceable.
;
-; TODO: LAA must not assume the AR fits in the deref region and should
+; LAA must not assume the AR fits in the deref region and should
; fall back to the wide low bound.
;
; Pseudocode:
@@ -90,10 +90,10 @@ define void @reverse_top_spills(ptr dereferenceable(16) %A, ptr dereferenceable(
; CHECK-NEXT: %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: (5 + %B)<nuw> High: (17 + %B))
+; CHECK-NEXT: (Low: null High: (17 + %B))
; CHECK-NEXT: Member: {(13 + %B)<nuw>,+,-4}<nw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: (5 + %A)<nuw> High: (17 + %A))
+; CHECK-NEXT: (Low: null High: (17 + %A))
; CHECK-NEXT: Member: {(13 + %A)<nuw>,+,-4}<nw><%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
diff --git a/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll b/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
index dbaaeadb6a890..172501d16e3b1 100644
--- a/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
+++ b/llvm/test/Transforms/LoopVectorize/reverse-loop-length.ll
@@ -59,26 +59,41 @@ exit:
ret ptr %res
}
-; TODO: The last iteration lands exactly on %ptr, so the accessed range [0, 16)
-; exactly fills the dereferenceable region, and the early-exit loop should be
+; The last iteration lands exactly on %ptr, so the accessed range [0, 16)
+; exactly fills the dereferenceable region, and the early-exit loop is
; vectorized.
define ptr @reverse_reaches_base_static_length(ptr dereferenceable(16) %ptr) {
; CHECK-LABEL: define ptr @reverse_reaches_base_static_length(
; CHECK-SAME: ptr dereferenceable(16) [[PTR:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 15, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[ELEMENT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[IV]]
-; CHECK-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[ELEMENT_GEP]], align 1
-; CHECK-NEXT: [[FOUND_CHECK:%.*]] = icmp eq i8 [[ELEMENT]], 0
-; CHECK-NEXT: br i1 [[FOUND_CHECK]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
-; CHECK-NEXT: [[RANGE_CHECK:%.*]] = icmp ne i64 [[IV]], 0
-; CHECK-NEXT: br i1 [[RANGE_CHECK]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 15, i64 14, i64 13, i64 12>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[PTR]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[REVERSE]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[ELEMENT_GEP]], %[[LOOP_HEADER]] ], [ null, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[TMP6]], %[[VECTOR_EARLY_EXIT]] ], [ null, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret ptr [[RES]]
;
entry:
More information about the llvm-commits
mailing list