[llvm] [LAA] Honor no-wrap predicates already in PSE in isNoWrap (PR #226430)
Tim Besard via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 23:59:53 PDT 2026
https://github.com/maleadt updated https://github.com/llvm/llvm-project/pull/226430
>From 23de74525333204048ade995a20a69cc0cd0d3ad Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Thu, 24 Sep 2026 13:02:22 +0200
Subject: [PATCH 1/2] [LoopLoadElim] Add tests for candidates needing LAA wrap
predicates (NFC)
Add store-to-load forwarding candidates whose load or store pointer LAA
can only prove non-wrapping with an NUSW SCEV predicate, which it adds
while analyzing the dependences of the loop. LLE currently fails to
forward them.
---
...wrap-predicate-from-dependence-analysis.ll | 101 ++++++++++++++++++
1 file changed, 101 insertions(+)
create mode 100644 llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
diff --git a/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
new file mode 100644
index 00000000000000..27758ccf27a8a9
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-load-elim -S %s | FileCheck %s
+
+; Null is a valid address in addrspace(1), so LAA cannot prove that the
+; pointers of the non-inbounds GEPs below do not wrap and adds NUSW predicates
+; for them while analyzing the dependence between the store to A[i] and the
+; load of A[i-1]. LLE must recognize these predicates as already present when
+; it checks the strides of the candidate and forward the stored value in the
+; versioned loop.
+
+define void @store_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
+; CHECK-LABEL: define void @store_ptr_needs_wrap_predicate(
+; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
+; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.prev = add nsw i64 %iv, -1
+ %gep.a.prev = getelementptr inbounds double, ptr addrspace(1) %A, i64 %iv.prev
+ %a.prev = load double, ptr addrspace(1) %gep.a.prev, align 8
+ %gep.b = getelementptr inbounds double, ptr addrspace(1) %B, i64 %iv.prev
+ %b = load double, ptr addrspace(1) %gep.b, align 8
+ %gep.a = getelementptr double, ptr addrspace(1) %A, i64 %iv
+ %a = load double, ptr addrspace(1) %gep.a, align 8
+ %mul = fmul double %b, %a.prev
+ %sub = fsub double %a, %mul
+ store double %sub, ptr addrspace(1) %gep.a, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %N
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @load_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
+; CHECK-LABEL: define void @load_ptr_needs_wrap_predicate(
+; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
+; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.prev = add nsw i64 %iv, -1
+ %gep.a.prev = getelementptr double, ptr addrspace(1) %A, i64 %iv.prev
+ %a.prev = load double, ptr addrspace(1) %gep.a.prev, align 8
+ %gep.b = getelementptr inbounds double, ptr addrspace(1) %B, i64 %iv.prev
+ %b = load double, ptr addrspace(1) %gep.b, align 8
+ %gep.a = getelementptr inbounds double, ptr addrspace(1) %A, i64 %iv
+ %a = load double, ptr addrspace(1) %gep.a, align 8
+ %mul = fmul double %b, %a.prev
+ %sub = fsub double %a, %mul
+ store double %sub, ptr addrspace(1) %gep.a, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %N
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 1e34dc39e81301e49c86bd6a03b65044a3c2a71d Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Thu, 24 Sep 2026 13:02:23 +0200
Subject: [PATCH 2/2] [LAA] Honor no-wrap predicates already in PSE in isNoWrap
getPtrStride is documented to check that the pointer does not wrap
assuming the predicate of PSE holds. Before #203787, isNoWrap did so via
PSE.hasNoOverflow, which remembered the NUSW flags added through
setNoOverflow. #203787 removed that bookkeeping, so a query that may not
add predicates now only accepts statically proven no-wrap facts and no
longer sees an NUSW predicate LAA already added for the same pointer.
LoopLoadElimination makes such queries for the load and store of each
forwarding candidate (isDependenceDistanceOfOne). If one of the pointers
needs a wrap predicate, e.g. a non-inbounds GEP in an address space
where null is a valid address, the candidate is rejected ("Pointer may
wrap"), even though the dependence analysis already added the predicate
and LLE versions the loop on all predicates of PSE anyway. Recurrences
like A[i] -= B[i-1] * A[i-1] then reload A[i-1] from memory in every
iteration instead of reusing the stored value.
If isNoWrap may not add a predicate, return whether the predicate of PSE
already implies the NUSW wrap predicate for the AddRec. Callers that may
add predicates are unchanged: they collect the predicate, and adding it
to PSE is a no-op if it is already implied.
This also restores the order of the runtime check groups in
retry-runtime-checks-after-dependence-analysis.ll that #203787 changed:
when the runtime checks are rebuilt after the dependence analysis, the
first pass without new assumptions again accepts pointers such as %gep.b
whose NUSW predicates were added when the checks were first built,
instead of deferring them to the second pass.
Assisted-by: Claude Code, Codex
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 13 ++-
...untime-checks-after-dependence-analysis.ll | 48 ++++-----
...wrap-predicate-from-dependence-analysis.ll | 97 ++++++++++++++++---
3 files changed, 113 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 00b67ed4b63f02..9c77a78336de2f 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1141,11 +1141,11 @@ isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr,
return true;
}
+ ScalarEvolution &SE = *PSE.getSE();
+ const SCEVPredicate *WrapPred =
+ SE.getWrapPredicate(AR, SCEVWrapPredicate::IncrementNUSW);
if (Ptr && Predicates) {
- ScalarEvolution &SE = *PSE.getSE();
- SCEVWrapPredicate::IncrementWrapFlags Flags =
- SCEVWrapPredicate::IncrementNUSW;
- Predicates->push_back(SE.getWrapPredicate(AR, Flags));
+ Predicates->push_back(WrapPred);
LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n"
<< "LAA: Pointer: " << *Ptr << "\n"
<< "LAA: SCEV: " << *AR << "\n"
@@ -1153,7 +1153,10 @@ isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr,
return true;
}
- return false;
+ // Without adding a new predicate, AR may still be known not to wrap if the
+ // predicates of PSE already imply it, e.g. because a wrap predicate for AR
+ // was added while analyzing the dependences of the loop.
+ return PSE.getPredicate().implies(WrapPred, SE);
}
static void visitPointers(Value *StartPtr, const Loop &InnermostLoop,
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
index 88b16d0f325347..63abd4ef70d632 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
@@ -72,27 +72,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_st
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
-; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
-; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b))
; CHECK-NEXT: Member: {%b,+,5}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
+; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -265,27 +265,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_may_wrap(ptr %a, pt
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
-; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
-; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b))
; CHECK-NEXT: Member: {%b,+,8}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
+; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -325,29 +325,29 @@ define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) {
; CHECK-NEXT: Run-time memory checks:
; CHECK-NEXT: Check 0:
; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: ptr %A
-; CHECK-NEXT: Against group GRP1:
; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: ptr %A
+; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT: ptr %A
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: ptr %A
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: %A High: (4 + %A))
-; CHECK-NEXT: Member: %A
-; CHECK-NEXT: Group GRP1:
; CHECK-NEXT: (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
; CHECK-NEXT: Member: {(100 + %A),+,8}<%loop>
-; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: Group GRP1:
; CHECK-NEXT: (Low: (100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A) High: (96 + (16 * (zext i32 %offset to i64))<nuw><nsw> + %A))
; CHECK-NEXT: Member: {(100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A),+,8}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %A High: (4 + %A))
+; CHECK-NEXT: Member: %A
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
diff --git a/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
index 27758ccf27a8a9..ef61d4bc269c21 100644
--- a/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
+++ b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
@@ -11,24 +11,57 @@
define void @store_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
; CHECK-LABEL: define void @store_ptr_needs_wrap_predicate(
; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr addrspace(1) [[A]], i64 8
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 8, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(1) [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr addrspace(1) [[TMP1]], [[SCEVGEP]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_PH_LVER_ORIG:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH_LVER_ORIG]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[LOOP_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
-; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[B_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A_LVER_ORIG:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[MUL_LVER_ORIG:%.*]] = fmul double [[B_LVER_ORIG]], [[A_PREV]]
+; CHECK-NEXT: [[SUB_LVER_ORIG:%.*]] = fsub double [[A_LVER_ORIG]], [[MUL_LVER_ORIG]]
+; CHECK-NEXT: store double [[SUB_LVER_ORIG]], ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC_LVER_ORIG:%.*]] = icmp eq i64 [[IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EC_LVER_ORIG]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load double, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], %[[LOOP_PH]] ], [ [[SUB:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 1, %[[LOOP_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV_PREV1:%.*]] = add nsw i64 [[IV1]], -1
+; CHECK-NEXT: [[GEP_A_PREV1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[A_PREV1:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV1]], align 8
+; CHECK-NEXT: [[GEP_B1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B1]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV1]]
; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[STORE_FORWARDED]]
+; CHECK-NEXT: [[SUB]] = fsub double [[A]], [[MUL]]
; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT2:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT2]]:
+; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -57,24 +90,56 @@ exit:
define void @load_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
; CHECK-LABEL: define void @load_ptr_needs_wrap_predicate(
; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -2
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 8, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(1) [[A]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr addrspace(1) [[TMP1]], [[A]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_PH_LVER_ORIG:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH_LVER_ORIG]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[LOOP_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
-; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[B_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[MUL_LVER_ORIG:%.*]] = fmul double [[B_LVER_ORIG]], [[A_PREV]]
+; CHECK-NEXT: [[SUB_LVER_ORIG:%.*]] = fsub double [[A_LVER_ORIG]], [[MUL_LVER_ORIG]]
+; CHECK-NEXT: store double [[SUB_LVER_ORIG]], ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC_LVER_ORIG:%.*]] = icmp eq i64 [[IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EC_LVER_ORIG]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load double, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], %[[LOOP_PH]] ], [ [[SUB:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 1, %[[LOOP_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV_PREV1:%.*]] = add nsw i64 [[IV1]], -1
+; CHECK-NEXT: [[GEP_A_PREV1:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[A_PREV1:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV1]], align 8
+; CHECK-NEXT: [[GEP_B1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B1]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV1]]
; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[STORE_FORWARDED]]
+; CHECK-NEXT: [[SUB]] = fsub double [[A]], [[MUL]]
; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT2:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT2]]:
+; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list