[llvm] [LAA] Honor no-wrap predicates already in PSE in isNoWrap (PR #226430)
Tim Besard via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 03:33:18 PDT 2026
https://github.com/maleadt created https://github.com/llvm/llvm-project/pull/226430
getPtrStride checks pointer wrapping assuming PSE's predicate holds. Before #203787, isNoWrap used `PSE.hasNoOverflow` to see NUSW flags recorded by `setNoOverflow`. Removing that bookkeeping made queries that cannot add predicates accept only static no-wrap facts, ignoring NUSW predicates LAA already added for the pointer.
LoopLoadElimination's `isDependenceDistanceOfOne` makes these queries for each forwarding candidate's load and store. A pointer needing a wrap predicate (e.g. a non-inbounds GEP where null is valid) is rejected with "Pointer may wrap", although dependence analysis already added the predicate and LLE versions the loop on all PSE predicates. Recurrences such as `A[i] -= B[i-1] * A[i-1]` then reload `A[i-1]` each iteration instead of reusing the stored value.
When isNoWrap cannot add predicates, check whether PSE's predicate implies the AddRec's NUSW wrap predicate. Callers allowed to add predicates are unchanged: they collect it, and adding an implied predicate to PSE is a no-op. The change introduces no new predicates.
This also restores the pre-#203787 check order in `retry-runtime-checks-after-dependence-analysis.ll`. When rebuilding checks after dependence analysis, the first pass again accepts pointers such as `%gep.b` with previously added NUSW predicates, rather than deferring them to the second pass.
Motivation: For `A[i] -= B[i-1] * A[i-1]; C[i] = ...` with may-aliasing `C` preventing GVN forwarding (addrspace(1), `opt -O3`, n=256), losing forwarding makes the loop 2.4x slower on Zen 5: LLVM 22 takes 318 ns, main 780 ns, and the patch 320 ns. Compile time is unchanged for a synthetic loop with 512 accesses needing wrap predicates (±1% instructions in `print<access-info>`).
Assisted by: Codex 5.5, Astra 6
Part of a series of patches developed while migrating Julia to LLVM 23
>From 310a4c72f55f542e3e1af4926b296c7959d4d237 Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Thu, 24 Sep 2026 13:02:22 +0200
Subject: [PATCH 1/2] [LoopLoadElim] Add tests for candidates needing LAA wrap
predicates (NFC)
Add store-to-load forwarding candidates whose load or store pointer LAA
can only prove non-wrapping with an NUSW SCEV predicate, which it adds
while analyzing the dependences of the loop. LLE currently fails to
forward them.
---
...wrap-predicate-from-dependence-analysis.ll | 101 ++++++++++++++++++
1 file changed, 101 insertions(+)
create mode 100644 llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
diff --git a/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
new file mode 100644
index 0000000000000..27758ccf27a8a
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-load-elim -S %s | FileCheck %s
+
+; Null is a valid address in addrspace(1), so LAA cannot prove that the
+; pointers of the non-inbounds GEPs below do not wrap and adds NUSW predicates
+; for them while analyzing the dependence between the store to A[i] and the
+; load of A[i-1]. LLE must recognize these predicates as already present when
+; it checks the strides of the candidate and forward the stored value in the
+; versioned loop.
+
+define void @store_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
+; CHECK-LABEL: define void @store_ptr_needs_wrap_predicate(
+; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
+; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.prev = add nsw i64 %iv, -1
+ %gep.a.prev = getelementptr inbounds double, ptr addrspace(1) %A, i64 %iv.prev
+ %a.prev = load double, ptr addrspace(1) %gep.a.prev, align 8
+ %gep.b = getelementptr inbounds double, ptr addrspace(1) %B, i64 %iv.prev
+ %b = load double, ptr addrspace(1) %gep.b, align 8
+ %gep.a = getelementptr double, ptr addrspace(1) %A, i64 %iv
+ %a = load double, ptr addrspace(1) %gep.a, align 8
+ %mul = fmul double %b, %a.prev
+ %sub = fsub double %a, %mul
+ store double %sub, ptr addrspace(1) %gep.a, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %N
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @load_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
+; CHECK-LABEL: define void @load_ptr_needs_wrap_predicate(
+; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
+; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.prev = add nsw i64 %iv, -1
+ %gep.a.prev = getelementptr double, ptr addrspace(1) %A, i64 %iv.prev
+ %a.prev = load double, ptr addrspace(1) %gep.a.prev, align 8
+ %gep.b = getelementptr inbounds double, ptr addrspace(1) %B, i64 %iv.prev
+ %b = load double, ptr addrspace(1) %gep.b, align 8
+ %gep.a = getelementptr inbounds double, ptr addrspace(1) %A, i64 %iv
+ %a = load double, ptr addrspace(1) %gep.a, align 8
+ %mul = fmul double %b, %a.prev
+ %sub = fsub double %a, %mul
+ store double %sub, ptr addrspace(1) %gep.a, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %N
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 420a75c628f819740e9cf4c750774c7946800f82 Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Thu, 24 Sep 2026 13:02:23 +0200
Subject: [PATCH 2/2] [LAA] Honor no-wrap predicates already in PSE in isNoWrap
getPtrStride is documented to check that the pointer does not wrap
assuming the predicate of PSE holds. Before #203787, isNoWrap did so via
PSE.hasNoOverflow, which remembered the NUSW flags added through
setNoOverflow. #203787 removed that bookkeeping, so a query that may not
add predicates now only accepts statically proven no-wrap facts and no
longer sees an NUSW predicate LAA already added for the same pointer.
LoopLoadElimination makes such queries for the load and store of each
forwarding candidate (isDependenceDistanceOfOne). If one of the pointers
needs a wrap predicate, e.g. a non-inbounds GEP in an address space
where null is a valid address, the candidate is rejected ("Pointer may
wrap"), even though the dependence analysis already added the predicate
and LLE versions the loop on all predicates of PSE anyway. Recurrences
like A[i] -= B[i-1] * A[i-1] then reload A[i-1] from memory in every
iteration instead of reusing the stored value.
If isNoWrap may not add a predicate, return whether the predicate of PSE
already implies the NUSW wrap predicate for the AddRec. Callers that may
add predicates are unchanged: they collect the predicate, and adding it
to PSE is a no-op if it is already implied.
This also restores the order of the runtime check groups in
retry-runtime-checks-after-dependence-analysis.ll that #203787 changed:
when the runtime checks are rebuilt after the dependence analysis, the
first pass without new assumptions again accepts pointers such as %gep.b
whose NUSW predicates were added when the checks were first built,
instead of deferring them to the second pass.
Assisted-by: Claude Code, Codex
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 7 +-
...untime-checks-after-dependence-analysis.ll | 48 ++++-----
...wrap-predicate-from-dependence-analysis.ll | 97 ++++++++++++++++---
3 files changed, 111 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 409d9ceb5b812..7a487072e6ad6 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1153,7 +1153,12 @@ isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr,
return true;
}
- return false;
+ // Without adding a new predicate, AR may still be known not to wrap if the
+ // predicates of PSE already imply it, e.g. because a wrap predicate for AR
+ // was added while analyzing the dependences of the loop.
+ ScalarEvolution &SE = *PSE.getSE();
+ return PSE.getPredicate().implies(
+ SE.getWrapPredicate(AR, SCEVWrapPredicate::IncrementNUSW), SE);
}
static void visitPointers(Value *StartPtr, const Loop &InnermostLoop,
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
index 88b16d0f32534..63abd4ef70d63 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
@@ -72,27 +72,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_st
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
-; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
-; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b))
; CHECK-NEXT: Member: {%b,+,5}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
+; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -265,27 +265,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_may_wrap(ptr %a, pt
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
+; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
; CHECK-NEXT: Group GRP1:
-; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
-; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
-; CHECK-NEXT: Group GRP2:
; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b))
; CHECK-NEXT: Member: {%b,+,8}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
+; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
@@ -325,29 +325,29 @@ define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) {
; CHECK-NEXT: Run-time memory checks:
; CHECK-NEXT: Check 0:
; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: ptr %A
-; CHECK-NEXT: Against group GRP1:
; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
; CHECK-NEXT: Check 1:
; CHECK-NEXT: Comparing group GRP0:
-; CHECK-NEXT: ptr %A
+; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
; CHECK-NEXT: Against group GRP2:
-; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT: ptr %A
; CHECK-NEXT: Check 2:
; CHECK-NEXT: Comparing group GRP1:
-; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
-; CHECK-NEXT: Against group GRP2:
; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: ptr %A
; CHECK-NEXT: Grouped accesses:
; CHECK-NEXT: Group GRP0:
-; CHECK-NEXT: (Low: %A High: (4 + %A))
-; CHECK-NEXT: Member: %A
-; CHECK-NEXT: Group GRP1:
; CHECK-NEXT: (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
; CHECK-NEXT: Member: {(100 + %A),+,8}<%loop>
-; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: Group GRP1:
; CHECK-NEXT: (Low: (100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A) High: (96 + (16 * (zext i32 %offset to i64))<nuw><nsw> + %A))
; CHECK-NEXT: Member: {(100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A),+,8}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %A High: (4 + %A))
+; CHECK-NEXT: Member: %A
; CHECK-EMPTY:
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
; CHECK-NEXT: SCEV assumptions:
diff --git a/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
index 27758ccf27a8a..ef61d4bc269c2 100644
--- a/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
+++ b/llvm/test/Transforms/LoopLoadElim/wrap-predicate-from-dependence-analysis.ll
@@ -11,24 +11,57 @@
define void @store_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
; CHECK-LABEL: define void @store_ptr_needs_wrap_predicate(
; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr addrspace(1) [[A]], i64 8
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 8, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(1) [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr addrspace(1) [[TMP1]], [[SCEVGEP]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_PH_LVER_ORIG:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH_LVER_ORIG]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[LOOP_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
-; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[B_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A_LVER_ORIG:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[MUL_LVER_ORIG:%.*]] = fmul double [[B_LVER_ORIG]], [[A_PREV]]
+; CHECK-NEXT: [[SUB_LVER_ORIG:%.*]] = fsub double [[A_LVER_ORIG]], [[MUL_LVER_ORIG]]
+; CHECK-NEXT: store double [[SUB_LVER_ORIG]], ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC_LVER_ORIG:%.*]] = icmp eq i64 [[IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EC_LVER_ORIG]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load double, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], %[[LOOP_PH]] ], [ [[SUB:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 1, %[[LOOP_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV_PREV1:%.*]] = add nsw i64 [[IV1]], -1
+; CHECK-NEXT: [[GEP_A_PREV1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[A_PREV1:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV1]], align 8
+; CHECK-NEXT: [[GEP_B1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B1]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV1]]
; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[STORE_FORWARDED]]
+; CHECK-NEXT: [[SUB]] = fsub double [[A]], [[MUL]]
; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT2:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT2]]:
+; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -57,24 +90,56 @@ exit:
define void @load_ptr_needs_wrap_predicate(ptr addrspace(1) noalias %A, ptr addrspace(1) noalias %B, i64 %N) {
; CHECK-LABEL: define void @load_ptr_needs_wrap_predicate(
; CHECK-SAME: ptr addrspace(1) noalias [[A:%.*]], ptr addrspace(1) noalias [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -2
+; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 8, i64 [[TMP0]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(1) [[A]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult ptr addrspace(1) [[TMP1]], [[A]]
+; CHECK-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_PH_LVER_ORIG:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH_LVER_ORIG]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[LOOP_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_PREV:%.*]] = add nsw i64 [[IV]], -1
; CHECK-NEXT: [[GEP_A_PREV:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV_PREV]]
; CHECK-NEXT: [[A_PREV:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV]], align 8
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV]]
-; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[B_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_B]], align 8
+; CHECK-NEXT: [[GEP_A_LVER_ORIG:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV]]
+; CHECK-NEXT: [[A_LVER_ORIG:%.*]] = load double, ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[MUL_LVER_ORIG:%.*]] = fmul double [[B_LVER_ORIG]], [[A_PREV]]
+; CHECK-NEXT: [[SUB_LVER_ORIG:%.*]] = fsub double [[A_LVER_ORIG]], [[MUL_LVER_ORIG]]
+; CHECK-NEXT: store double [[SUB_LVER_ORIG]], ptr addrspace(1) [[GEP_A_LVER_ORIG]], align 8
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC_LVER_ORIG:%.*]] = icmp eq i64 [[IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EC_LVER_ORIG]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: [[LOAD_INITIAL:%.*]] = load double, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[STORE_FORWARDED:%.*]] = phi double [ [[LOAD_INITIAL]], %[[LOOP_PH]] ], [ [[SUB:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 1, %[[LOOP_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV_PREV1:%.*]] = add nsw i64 [[IV1]], -1
+; CHECK-NEXT: [[GEP_A_PREV1:%.*]] = getelementptr double, ptr addrspace(1) [[A]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[A_PREV1:%.*]] = load double, ptr addrspace(1) [[GEP_A_PREV1]], align 8
+; CHECK-NEXT: [[GEP_B1:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[B]], i64 [[IV_PREV1]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr addrspace(1) [[GEP_B1]], align 8
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[A]], i64 [[IV1]]
; CHECK-NEXT: [[A:%.*]] = load double, ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[A_PREV]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub double [[A]], [[MUL]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[B]], [[STORE_FORWARDED]]
+; CHECK-NEXT: [[SUB]] = fsub double [[A]], [[MUL]]
; CHECK-NEXT: store double [[SUB]], ptr addrspace(1) [[GEP_A]], align 8
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT2:.*]], label %[[LOOP1]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT2]]:
+; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list